<commit_before>﻿using System.Collections.Generic;
using System.IO;
using System.Linq;
using Ghpr.Core.Common;
using Newtonsoft.Json;

namespace Ghpr.Core.Helpers
{
    public static class ItemInfoHelper
    {
        public static void SaveItemInfo(string path, string filename, ItemInfo itemInfo, bool removeExisting = true)
        {
            var serializer = new JsonSerializer();
            if (!Directory.Exists(path))
            {
                Directory.CreateDirectory(path);
            }
            var fullRunsPath = Path.Combine(path, filename);
            if (!File.Exists(fullRunsPath))
            {
                var items = new List<ItemInfo>
                {
                    itemInfo
                };
                using (var file = File.CreateText(fullRunsPath))
                {
                    serializer.Serialize(file, items);
                }
            }
            else
            {
                List<ItemInfo> items;
                using (var file = File.OpenText(fullRunsPath))
                {
                    items = (List<ItemInfo>)serializer.Deserialize(file, typeof(List<ItemInfo>));
                    if (removeExisting && items.Any(i => i.Guid.Equals(itemInfo.Guid)))
                    {
                        items.Remove(items.First(i => i.Guid.Equals(itemInfo.Guid)));
                    }
                    items.Add(itemInfo);
                }
                using (var file = File.CreateText(fullRunsPath))
                {
                    items = items.OrderByDescending(x => x.Start).ToList();
                    serializer.Serialize(file, items);
                }
            }
        }
    }
}<commit_msg>remove existing fix<commit_after>﻿using System.Collections.Generic;
using System.IO;
using System.Linq;
using Ghpr.Core.Common;
using Newtonsoft.Json;

namespace Ghpr.Core.Helpers
{
    public static class ItemInfoHelper
    {
        public static void SaveItemInfo(string path, string filename, ItemInfo itemInfo, bool removeExisting = true)
        {
            var serializer = new JsonSerializer();
            if (!Directory.Exists(path))
            {
                Directory.CreateDirectory(path);
            }
            var fullRunsPath = Path.Combine(path, filename);
            if (!File.Exists(fullRunsPath))
            {
                var items = new List<ItemInfo>
                {
                    itemInfo
                };
                using (var file = File.CreateText(fullRunsPath))
                {
                    serializer.Serialize(file, items);
                }
            }
            else
            {
                List<ItemInfo> items;
                using (var file = File.OpenText(fullRunsPath))
                {
                    items = (List<ItemInfo>)serializer.Deserialize(file, typeof(List<ItemInfo>));
                    if (removeExisting && items.Any(i => i.Guid.Equals(itemInfo.Guid)))
                    {
                        var existingItems = items.Where(i => i.Guid.Equals(itemInfo.Guid));
                        foreach (var item in existingItems)
                        {
                            items.Remove(item);
                        }
                    }
                    items.Add(itemInfo);
                }
                using (var file = File.CreateText(fullRunsPath))
                {
                    items = items.OrderByDescending(x => x.Start).ToList();
                    serializer.Serialize(file, items);
                }
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.09313998884551032, 'fraction_numerical': 0.0, 'mean_word_length': 1.2474937343358397, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6065899', 'n_tokens_mistral': 909, 'n_tokens_neox': 862, 'n_words': 166}