<commit_before>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.RegularExpressions;
using Sep.Git.Tfs.Core;
using Sep.Git.Tfs.Core.TfsInterop;

namespace Sep.Git.Tfs.Util
{
    public enum ChangeType
    {
        Update,
        Delete,
    }

    public class ApplicableChange
    {
        public ChangeType Type { get; set; }
        public string GitPath { get; set; }

        public static ApplicableChange Update(string path)
        {
            return new ApplicableChange { Type = ChangeType.Update, GitPath = path };
        }

        public static ApplicableChange Delete(string path)
        {
            return new ApplicableChange { Type = ChangeType.Delete, GitPath = path };
        }
    }

    public class ChangeSieve
    {
        readonly IDictionary<string, GitObject> _initialTree;
        readonly IChangeset _changeset;
        readonly IGitTfsRemote _remote;

        public ChangeSieve(IDictionary<string, GitObject> initialTree, IChangeset changeset, IGitTfsRemote remote)
        {
            _initialTree = initialTree;
            _changeset = changeset;
            _remote = remote;
        }

        public bool HasChanges()
        {
            return FilteredChanges.Any();
        }

        public IEnumerable<IChange> ChangesToFetch()
        {
            return FilteredChanges.Select(c => c.Change);
        }

        public IEnumerable<IChange> ChangesToApply()
        {
            return Sort(ChangesToFetch());
        }

        public IEnumerable<ApplicableChange> ChangesToApply2()
        {
            var compartments = new {
                Deleted = new List<ApplicableChange>(),
                Updated = new List<ApplicableChange>(),
            };
            foreach (var change in NamedChanges)
            {
                if (change.Change.ChangeType.IncludesOneOf(TfsChangeType.Delete))
                {
                    if(change.GitPath != null)
                        compartments.Deleted.Add(ApplicableChange.Delete(change.GitPath));
                }
                else if (change.Change.ChangeType.IncludesOneOf(TfsChangeType.Rename))
                {
                    var oldPath = GetPathInGitRepo(GetPathBeforeRename(change.Change.Item));
                    if (oldPath != null)
                        compartments.Deleted.Add(ApplicableChange.Delete(oldPath));
                    if (change.GitPath != null && !_remote.ShouldSkip(change.GitPath))
                        compartments.Updated.Add(ApplicableChange.Update(change.GitPath));
                }
                else
                {
                    if (change.GitPath != null && !_remote.ShouldSkip(change.GitPath))
                        compartments.Updated.Add(ApplicableChange.Update(change.GitPath));
                }
            }
            return compartments.Deleted.Concat(compartments.Updated);
        }

        private IEnumerable<IChange> Sort(IEnumerable<IChange> changes)
        {
            return changes.OrderBy(change => Rank(change.ChangeType));
        }

        private int Rank(TfsChangeType type)
        {
            if (type.IncludesOneOf(TfsChangeType.Delete))
                return 0;
            if (type.IncludesOneOf(TfsChangeType.Rename))
                return 1;
            return 2;
        }

        class NamedChange
        {
            public string GitPath { get; set; }
            public IChange Change { get; set; }
        }

        IEnumerable<NamedChange> NamedChanges
        {
            get
            {
                return _changeset.Changes.Select(c => new NamedChange { GitPath = GetPathInGitRepo(c.Item.ServerItem), Change = c });
            }
        }

        IEnumerable<NamedChange> FilteredChanges
        {
            get
            {
                return NamedChanges.Where(c => c.GitPath != null && !_remote.ShouldSkip(c.GitPath));
            }
        }<commit_msg>ChangeSieve tests pass.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.RegularExpressions;
using Sep.Git.Tfs.Core;
using Sep.Git.Tfs.Core.TfsInterop;

namespace Sep.Git.Tfs.Util
{
    public enum ChangeType
    {
        Update,
        Delete,
    }

    public class ApplicableChange
    {
        public ChangeType Type { get; set; }
        public string GitPath { get; set; }

        public static ApplicableChange Update(string path)
        {
            return new ApplicableChange { Type = ChangeType.Update, GitPath = path };
        }

        public static ApplicableChange Delete(string path)
        {
            return new ApplicableChange { Type = ChangeType.Delete, GitPath = path };
        }
    }

    public class ChangeSieve
    {
        readonly IDictionary<string, GitObject> _initialTree;
        readonly IChangeset _changeset;
        readonly IGitTfsRemote _remote;

        public ChangeSieve(IDictionary<string, GitObject> initialTree, IChangeset changeset, IGitTfsRemote remote)
        {
            _initialTree = initialTree;
            _changeset = changeset;
            _remote = remote;
        }

        public bool HasChanges()
        {
            return FilteredChanges.Any();
        }

        public IEnumerable<IChange> ChangesToFetch()
        {
            return FilteredChanges.Select(c => c.Change);
        }

        public IEnumerable<IChange> ChangesToApply()
        {
            return Sort(ChangesToFetch());
        }

        public IEnumerable<ApplicableChange> ChangesToApply2()
        {
            var compartments = new {
                Deleted = new List<ApplicableChange>(),
                Updated = new List<ApplicableChange>(),
            };
            foreach (var change in NamedChanges)
            {
                if (change.Change.Item.ItemType == TfsItemType.File)
                {
                    if (change.Change.ChangeType.IncludesOneOf(TfsChangeType.Delete))
                    {
                        if (change.GitPath != null)
                            compartments.Deleted.Add(ApplicableChange.Delete(change.GitPath));
                    }
                    else if (change.Change.ChangeType.IncludesOneOf(TfsChangeType.Rename))
                    {
                        var oldPath = GetPathInGitRepo(GetPathBeforeRename(change.Change.Item));
                        if (oldPath != null)
                            compartments.Deleted.Add(ApplicableChange.Delete(oldPath));
                        if (Include(change))
                            compartments.Updated.Add(ApplicableChange.Update(change.GitPath));
                    }
                    else
                    {
                        if (Include(change))
                            compartments.Updated.Add(ApplicableChange.Update(change.GitPath));
                    }
                }
            }
            return compartments.Deleted.Concat(compartments.Updated);
        }

        private IEnumerable<IChange> Sort(IEnumerable<IChange> changes)
        {
            return changes.OrderBy(change => Rank(change.ChangeType));
        }

        private int Rank(TfsChangeType type)
        {
            if (type.IncludesOneOf(TfsChangeType.Delete))
                return 0;
            if (type.IncludesOneOf(TfsChangeType.Rename))
                return 1;
            return 2;
        }

        class NamedChange
        {
            public string GitPath { get; set; }
            public IChange Change { get; set; }
        }

        IEnumerable<NamedChange> NamedChanges
        {
            get
            {
                return _changeset.Changes.Select(c => new NamedChange { GitPath = GetPathInGitRepo(c.Item.ServerItem), Change = c });
            }
        }

        [Obsolete("Inline this into ChangesToFetch()")]
        IEnumerable<NamedChange> FilteredChanges
        {
            get
            {
                return NamedChanges.Where(c => Include(c.GitPath));
            }
        }

        private bool Include(NamedChange change)
        {
            return Include(change.GitPath) && change.Change.Item.DeletionId == 0;
        }

        private bool Include(string pathInGitRepo)
        {
            return !String.IsNullOrEmpty(pathInGitRepo) && !_remote.ShouldSkip(pathInGitRepo);
        }
{'fraction_non_alphanumeric': 0.08361124344926155, 'fraction_numerical': 0.0010719390185802763, 'mean_word_length': 1.6183348924228251, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3277062', 'n_tokens_mistral': 2266, 'n_tokens_neox': 2133, 'n_words': 415}