<commit_before>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;

namespace QuickTag.Data
{
	public sealed class Folder : ISerializable
	{
		private List<ImageData> images;
		private ImageDataComparer comparer;

		public string Name { get; set; }
		public string Path { get; set; }

		public int Count
		{
			get
			{
				return this.images.Count;
			}
		}

		public bool IsReadOnly
		{
			get
			{
				return false;
			}
		}

		public ImageData this[int index]
		{
			get
			{
				return this.images[index];
			}
		}

		public ImageData this[string filePath]
		{
			get
			{
				return this.images.Where(i => i.ImagePath == filePath).FirstOrDefault();
			}
		}

		private Folder()
		{
			this.images = new List<ImageData>();
			this.comparer = new ImageDataComparer();
		}

		public void Add(ImageData item)
		{
			if (this.images.Count == 0)
			{
				this.images.Add(item);
				return;
			}

			var index = this.images.BinarySearch(item, this.comparer);
			if (index < 0)
			{
				this.images.Insert(~index, item);
			}
		}

		public void Clear()
		{
			this.images.Clear();
		}

		public bool Contains(ImageData item)
		{
			return this.images.Contains(item);
		}

		public void CopyTo(ImageData[] array, int index)
		{
			this.images.CopyTo(array, index);
		}

		public IEnumerator<ImageData> GetEnumerator()
		{
			return this.images.GetEnumerator();
		}

		public int IndexOf(ImageData item)
		{
			return this.images.IndexOf(item);
		}

		public bool Remove(ImageData item)
		{
			return this.images.Remove(item);
		}

		public void RemoveAt(int index)
		{
			this.images.RemoveAt(index);
		}

		public ImageData Search(string imagePath)
		{
			var matches = this.images.Where(i => i.ImagePath == imagePath);
			return matches.FirstOrDefault();
		}

		public void Update()
		{
			var extensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase){ ".bmp", ".jpg", ".jpeg", ".gif", ".png", ".tiff", ".svg", ".webm" };

			var currentFileNames = this.images.Select(i => i.ImagePath);
			var newFileNames = Directory.EnumerateFiles(this.Path, "*", SearchOption.AllDirectories).Where(f => extensions.Contains(System.IO.Path.GetExtension(f)));
			var newFilesInFolder = currentFileNames.Intersect(newFileNames); // all existing files are already in currentFileNames
			var deletedFiles = currentFileNames.Union(newFileNames); // all deleted files are not in newFileNames
			var deletedImageData = this.images.Where(i => deletedFiles.Contains(i.ImagePath));

			// Where you left off: newFilesInFolder needs to be all the items in newFileNames NOT in currentFileNames
			// deletedFiles needs to be all the items in currentFileNames NOT in newFileNames
			// you could probably use Enumerable.Except();

			// Add the new files to the folder.
			foreach (string newFile in newFilesInFolder)
			{
				this.images.Add(new ImageData(newFile));
			}

			// Remove the deleted files from the folder.
			foreach (ImageData data in deletedImageData)
			{
				this.Remove(data);
			}
		}

		public Dictionary<string, int> GetTags()
		{
			if (this.images.Count == 0)
			{
				return new Dictionary<string, int>();
			}

			Dictionary<string, int> result = new Dictionary<string, int>();
			foreach (ImageData data in this.images)<commit_msg>Added the other half of the folder update code<commit_after>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;

namespace QuickTag.Data
{
	public sealed class Folder : ISerializable
	{
		private List<ImageData> images;
		private ImageDataComparer comparer;

		public string Name { get; set; }
		public string Path { get; set; }

		public int Count
		{
			get
			{
				return this.images.Count;
			}
		}

		public bool IsReadOnly
		{
			get
			{
				return false;
			}
		}

		public ImageData this[int index]
		{
			get
			{
				return this.images[index];
			}
		}

		public ImageData this[string filePath]
		{
			get
			{
				return this.images.Where(i => i.ImagePath == filePath).FirstOrDefault();
			}
		}

		private Folder()
		{
			this.images = new List<ImageData>();
			this.comparer = new ImageDataComparer();
		}

		public void Add(ImageData item)
		{
			if (this.images.Count == 0)
			{
				this.images.Add(item);
				return;
			}

			var index = this.images.BinarySearch(item, this.comparer);
			if (index < 0)
			{
				this.images.Insert(~index, item);
			}
		}

		public void Clear()
		{
			this.images.Clear();
		}

		public bool Contains(ImageData item)
		{
			return this.images.Contains(item);
		}

		public void CopyTo(ImageData[] array, int index)
		{
			this.images.CopyTo(array, index);
		}

		public IEnumerator<ImageData> GetEnumerator()
		{
			return this.images.GetEnumerator();
		}

		public int IndexOf(ImageData item)
		{
			return this.images.IndexOf(item);
		}

		public bool Remove(ImageData item)
		{
			return this.images.Remove(item);
		}

		public void RemoveAt(int index)
		{
			this.images.RemoveAt(index);
		}

		public ImageData Search(string imagePath)
		{
			var matches = this.images.Where(i => i.ImagePath == imagePath);
			return matches.FirstOrDefault();
		}

		public void Update()
		{
			var extensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase){ ".bmp", ".jpg", ".jpeg", ".gif", ".png", ".tiff", ".svg", ".webm" };

			var currentFileNames = this.images.Select(i => i.ImagePath).ToList();
			var newFileNames = Directory.EnumerateFiles(this.Path, "*", SearchOption.AllDirectories).Where(f => extensions.Contains(System.IO.Path.GetExtension(f))).ToList();
			var newFilesInFolder = newFileNames.Except(currentFileNames).ToList();

			// Add the new files to the folder.
			foreach (string newFile in newFilesInFolder)
			{
				this.images.Add(new ImageData(newFile));
			}

			var deletedFiles = currentFileNames.Except(newFileNames).ToList();
			List<ImageData> deletedImageData = new List<ImageData>(deletedFiles.Count);
			foreach (string deletedFile in deletedFiles)
			{
				deletedImageData.Add(this.images.Where(i => i.ImagePath == deletedFile).FirstOrDefault());
			}

			// Remove the deleted files from the folder.
			foreach (ImageData data in deletedImageData)
			{
				this.Remove(data);
			}
		}

		public Dictionary<string, int> GetTags()
		{
			if (this.images.Count == 0)
			{
				return new Dictionary<string, int>();
			}

			Dictionary<string, int> result = new Dictionary<string, int>();
			foreach (ImageData data in this.images)
{'fraction_non_alphanumeric': 0.12111144669284205, 'fraction_numerical': 0.000906070673512534, 'mean_word_length': 3.342950819672131, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '341115', 'n_tokens_mistral': 2529, 'n_tokens_neox': 2109, 'n_words': 516}