<commit_before>﻿using Newtonsoft.Json;
using SensusService.Exceptions;
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;

namespace SensusService.DataStores.Local
{
    public class FileLocalDataStore : LocalDataStore
    {
        /// <summary>
        /// File being written with local data.
        /// </summary>
        private StreamWriter _file;

        /// <summary>
        /// Contains the path to the file currently being written.
        /// </summary>
        private string _path;

        /// <summary>
        /// Settings for serializing this data store.
        /// </summary>
        private readonly JsonSerializerSettings _serializationSettings = new JsonSerializerSettings
        {
            PreserveReferencesHandling = PreserveReferencesHandling.Objects,
            ReferenceLoopHandling = ReferenceLoopHandling.Serialize,
            TypeNameHandling = TypeNameHandling.All,
            ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor
        };

        private string StorageDirectory
        {
            get
            {
                string directory = Path.Combine(Protocol.StorageDirectory, GetType().FullName);

                if (!Directory.Exists(directory))
                    Directory.CreateDirectory(directory);

                return directory;
            }
        }

        protected override string DisplayName
        {
            get { return "File"; }
        }
        
        [JsonIgnore]
        public override bool Clearable
        {
            get { return true; }
        }

        public override void Start()
        {
            // file needs to be ready to accept data immediately
            lock (this)
                InitializeFile();

            base.Start();
        }

        protected override ICollection<Datum> CommitData(ICollection<Datum> data)
        {
            lock (this)
            {
                List<Datum> committedData = new List<Datum>();

                foreach (Datum datum in data)
                {
                    string datumJSON = null;
                    try { datumJSON = GetJSON(datum); }
                    catch (Exception ex) { SensusServiceHelper.Get().Logger.Log("Failed to get JSON for datum:  " + ex.Message, LoggingLevel.Normal); }

                    if (datumJSON != null)
                    {
                        bool writtenToFile = false;
                        try
                        {
                            _file.WriteLine(datumJSON);
                            writtenToFile = true;
                        }
                        catch (Exception ex)
                        {
                            SensusServiceHelper.Get().Logger.Log("Failed to write datum JSON to local file:  " + ex.Message, LoggingLevel.Normal);

                            try
                            {
                                InitializeFile();

                                SensusServiceHelper.Get().Logger.Log("Initialized new local file.", LoggingLevel.Normal);
                            }
                            catch (Exception ex2) { SensusServiceHelper.Get().Logger.Log("Failed to initialize new local data store file after failing to write the old one:  " + ex2.Message, LoggingLevel.Normal); }
                        }

                        if (writtenToFile)
                            committedData.Add(datum);
                    }
                }

                return committedData;
            }
        }

        public override ICollection<Datum> GetDataForRemoteDataStore()
        {
            lock (this)
            {
                CloseFile();

                // get local data from all files
                List<Datum> localData = new List<Datum>();
                foreach (string path in Directory.GetFiles(StorageDirectory))
                    try
                    {
                        using (StreamReader file = new StreamReader(path))
                        {
                            string line;
                            while ((line = file.ReadLine()) != null)
                                localData.Add(JsonConvert.DeserializeObject<Datum>(line, _serializationSettings));

                            file.Close();
                        }
                    }
                    catch (Exception ex) { SensusServiceHelper.Get().Logger.Log("Exception while reading local data store for transfer to remote:  " + ex.Message, LoggingLevel.Normal); }

                // this method is called from a remote data store, which doesn't care whether this local data store is running. don't start a new file unless this local data store is, in fact, running.
                if (Running)
                    InitializeFile();

                return localData;
            }
        }

        public override void ClearDataCommittedToRemoteDataStore(ICollection<Datum> dataCommittedToRemote)
        {
            lock (this)
            {
                SensusServiceHelper.Get().Logger.Log("Local data store received " + dataCommittedToRemote.Count + " remote-committed elements to clear.", LoggingLevel.Verbose);

                if (dataCommittedToRemote.Count == 0)
                    return;

                HashSet<Datum> hashDataCommittedToRemote = new HashSet<Datum>(dataCommittedToRemote);  // for quick access via hashing

                foreach (string path in Directory.GetFiles(StorageDirectory))
                    if (path != _path)  // don't process the file that's currently being written<commit_msg>Don't short-circuit processing of committed data when none is given, since doing so leaves files around.<commit_after>﻿using Newtonsoft.Json;
using SensusService.Exceptions;
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;

namespace SensusService.DataStores.Local
{
    public class FileLocalDataStore : LocalDataStore
    {
        /// <summary>
        /// File being written with local data.
        /// </summary>
        private StreamWriter _file;

        /// <summary>
        /// Contains the path to the file currently being written.
        /// </summary>
        private string _path;

        /// <summary>
        /// Settings for serializing this data store.
        /// </summary>
        private readonly JsonSerializerSettings _serializationSettings = new JsonSerializerSettings
        {
            PreserveReferencesHandling = PreserveReferencesHandling.Objects,
            ReferenceLoopHandling = ReferenceLoopHandling.Serialize,
            TypeNameHandling = TypeNameHandling.All,
            ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor
        };

        private string StorageDirectory
        {
            get
            {
                string directory = Path.Combine(Protocol.StorageDirectory, GetType().FullName);

                if (!Directory.Exists(directory))
                    Directory.CreateDirectory(directory);

                return directory;
            }
        }

        protected override string DisplayName
        {
            get { return "File"; }
        }
        
        [JsonIgnore]
        public override bool Clearable
        {
            get { return true; }
        }

        public override void Start()
        {
            // file needs to be ready to accept data immediately
            lock (this)
                InitializeFile();

            base.Start();
        }

        protected override ICollection<Datum> CommitData(ICollection<Datum> data)
        {
            lock (this)
            {
                List<Datum> committedData = new List<Datum>();

                foreach (Datum datum in data)
                {
                    string datumJSON = null;
                    try { datumJSON = GetJSON(datum); }
                    catch (Exception ex) { SensusServiceHelper.Get().Logger.Log("Failed to get JSON for datum:  " + ex.Message, LoggingLevel.Normal); }

                    if (datumJSON != null)
                    {
                        bool writtenToFile = false;
                        try
                        {
                            _file.WriteLine(datumJSON);
                            writtenToFile = true;
                        }
                        catch (Exception ex)
                        {
                            SensusServiceHelper.Get().Logger.Log("Failed to write datum JSON to local file:  " + ex.Message, LoggingLevel.Normal);

                            try
                            {
                                InitializeFile();

                                SensusServiceHelper.Get().Logger.Log("Initialized new local file.", LoggingLevel.Normal);
                            }
                            catch (Exception ex2) { SensusServiceHelper.Get().Logger.Log("Failed to initialize new local data store file after failing to write the old one:  " + ex2.Message, LoggingLevel.Normal); }
                        }

                        if (writtenToFile)
                            committedData.Add(datum);
                    }
                }

                return committedData;
            }
        }

        public override ICollection<Datum> GetDataForRemoteDataStore()
        {
            lock (this)
            {
                CloseFile();

                // get local data from all files
                List<Datum> localData = new List<Datum>();
                foreach (string path in Directory.GetFiles(StorageDirectory))
                    try
                    {
                        using (StreamReader file = new StreamReader(path))
                        {
                            string line;
                            while ((line = file.ReadLine()) != null)
                                localData.Add(JsonConvert.DeserializeObject<Datum>(line, _serializationSettings));

                            file.Close();
                        }
                    }
                    catch (Exception ex) { SensusServiceHelper.Get().Logger.Log("Exception while reading local data store for transfer to remote:  " + ex.Message, LoggingLevel.Normal); }

                // this method is called from a remote data store, which doesn't care whether this local data store is running. don't start a new file unless this local data store is, in fact, running.
                if (Running)
                    InitializeFile();

                return localData;
            }
        }

        public override void ClearDataCommittedToRemoteDataStore(ICollection<Datum> dataCommittedToRemote)
        {
            lock (this)
            {
                SensusServiceHelper.Get().Logger.Log("Local data store received " + dataCommittedToRemote.Count + " remote-committed elements to clear.", LoggingLevel.Verbose);

                HashSet<Datum> hashDataCommittedToRemote = new HashSet<Datum>(dataCommittedToRemote);  // for quick access via hashing

                foreach (string path in Directory.GetFiles(StorageDirectory))
                    if (path != _path)  // don't process the file that's currently being written
{'fraction_non_alphanumeric': 0.07448888888888888, 'fraction_numerical': 0.00044444444444444447, 'mean_word_length': 1.3862142099681867, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6434453', 'n_tokens_mistral': 2580, 'n_tokens_neox': 2431, 'n_words': 678}