<commit_before>﻿
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;

namespace Serenity.IO
{
    /// <summary>
    /// A fast glob implementation, compatible with .gitignore patterns of GIT
    /// </summary>
    public class GlobFilter
    {
        private GlobFilter excludeFilter;
        private HashSet<string> extensions;
        private List<string> startsWith;
        private List<string> endsWith;
        private List<string> contains;
        private HashSet<string> exactMatch;
        private List<Tuple<string, bool, string>> startsWithAndEndsWith;
        private List<Tuple<string, bool, string>> containsAndEndsWith;

        private bool isActive;
        private List<Func<string, bool>> matchers;

        private char[] DotAsteriskSlashBackQue = new char[] { '.', '*', '/', '\\', '?' };
        private char[] AsteriskSlashBackQue = new char[] { '*', '/', '\\', '?' };
        private char[] AsteriskQue = new char[] { '*', '?' };
        private char[] FolderSeps = new char[] { '\\', '/' };

        /// <summary>
        /// Creates a new GlobFilter, containing both include and exclude patterns.
        /// When "include" is null / empty, all files are included by default, unless
        /// they match a pattern in "exclude" list.
        /// When "exclude" is null, it is ignored.
        /// </summary>
        /// <param name="include">List of include patterns</param>
        /// <param name="exclude">List of exclude patterns</param>
        public GlobFilter(IEnumerable<string> include, IEnumerable<string> exclude)
            : this(include)
        {
            if (exclude != null)
            {
                var excludes = exclude.Where(x => !string.IsNullOrEmpty(x));
                if (excludes.Any())
                    this.excludeFilter = new GlobFilter(excludes);
            }
        }

        /// <summary>
        /// Creates a new GlobFilter, with just "include" globs.
        /// </summary>
        /// <param name="globs">List of patterns</param>
        public GlobFilter(IEnumerable<string> globs)
        {
            this.matchers = new List<Func<string, bool>>();

            if (globs == null)
                return;

            foreach (var glob in globs)
            {
                if (string.IsNullOrEmpty(glob))
                    continue;

                isActive = true;

                var sep = System.IO.Path.DirectorySeparatorChar;

                var s = glob;
                if (sep == '\\')
                    s = glob.Replace('/', '\\');

                var starDotIndex = s.IndexOf("*.");

                // exact match
                if (s[0] == sep &&
                    s[s.Length - 1] != '/' &&
                    s[s.Length - 1] != '\\' &&
                    s.IndexOfAny(AsteriskQue, 1) < 0)
                {
                    if (exactMatch == null)
                        exactMatch = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

                    exactMatch.Add(s.Substring(1));
                    continue;
                }

                // simple extension filter (*.txt)
                if (starDotIndex == 0 &&
                    s.IndexOfAny(DotAsteriskSlashBackQue, 2) < 0)
                {
                    if (extensions == null)
                        extensions = new HashSet<string>();

                    extensions.Add(s.Substring(1));
                    continue;
                }

                // simple extension filter with extra dot (*.csproj.user)
                if (starDotIndex == 0 &&
                    s.IndexOfAny(AsteriskSlashBackQue, 2) < 0)
                {
                    if (endsWith == null)
                        endsWith = new List<string>();

                    endsWith.Add(s.Substring(1));
                    continue;
                }

                // subdirectory under root and extension (/bin/some/*.txt)
                if (starDotIndex > 1 &&
                    s[starDotIndex - 1] == sep &&
                    s[0] == sep &&
                    s.IndexOfAny(AsteriskSlashBackQue, starDotIndex + 2) < 0 &&
                    s.LastIndexOfAny(AsteriskQue, starDotIndex - 1) < 0)
                {
                    if (startsWithAndEndsWith == null)
                        startsWithAndEndsWith = new List<Tuple<string, bool, string>>();

                    startsWithAndEndsWith.Add(new Tuple<string, bool, string>(
                        s.Substring(1, starDotIndex - 1), false, s.Substring(starDotIndex + 1)));

                    continue;
                }

                // extension under root (/*.txt)
                if (starDotIndex == 1 &&
                    s[0] == sep &&
                    s.IndexOfAny(AsteriskSlashBackQue, starDotIndex + 2) < 0)
                {
                    if (startsWithAndEndsWith == null)
                        startsWithAndEndsWith = new List<Tuple<string, bool, string>>();

                    startsWithAndEndsWith.Add(new Tuple<string, bool, string>(null, false, s.Substring(starDotIndex + 1)));
                    continue;
                }

                // directory filter at any depth (.git/)
                if (s[s.Length - 1] == sep &&
                    s[0] != sep &&
                    s.IndexOfAny(AsteriskQue, 1) < 0)
                {
                    if (contains == null)
                        contains = new List<string>();

                    if (startsWith == null)
                        startsWith = new List<string>();

                    contains.Add(sep.ToString() + s);
                    startsWith.Add(s);

                    continue;
                }

                // directory filter at root (/Imports/)
                if (s[s.Length - 1] == sep &&
                    s[0] == sep &&
                    s.IndexOfAny(AsteriskQue) < 0)
                {
                    startsWith.Add(s.Substring(1));
                    continue;
                }

                // folder than any then extension (App_Data/**/*.log) or (/App_Code/**/*.xyz)
                if (starDotIndex > 4 &&
                    s[starDotIndex - 1] == sep &&
                    s[starDotIndex - 2] == '*' &&
                    s[starDotIndex - 3] == '*' &&
                    s[starDotIndex - 4] == sep &&
                    s.LastIndexOfAny(AsteriskQue, starDotIndex - 5) < 0 &&
                    s.IndexOfAny(AsteriskSlashBackQue, starDotIndex + 2) < 0)
                {
                    if (startsWithAndEndsWith == null)
                        startsWithAndEndsWith = new List<Tuple<string, bool, string>>();

                    if (s[0] == sep)
                    {
                        startsWithAndEndsWith.Add(new Tuple<string, bool, string>(
                            s.Substring(1, starDotIndex - 4), true, s.Substring(starDotIndex + 1)));
                    }
                    else
                    {<commit_msg>resolve possible null reference exception in GlobFilter<commit_after>﻿
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;

namespace Serenity.IO
{
    /// <summary>
    /// A fast glob implementation, compatible with .gitignore patterns of GIT
    /// </summary>
    public class GlobFilter
    {
        private GlobFilter excludeFilter;
        private HashSet<string> extensions;
        private List<string> startsWith;
        private List<string> endsWith;
        private List<string> contains;
        private HashSet<string> exactMatch;
        private List<Tuple<string, bool, string>> startsWithAndEndsWith;
        private List<Tuple<string, bool, string>> containsAndEndsWith;

        private bool isActive;
        private List<Func<string, bool>> matchers;

        private char[] DotAsteriskSlashBackQue = new char[] { '.', '*', '/', '\\', '?' };
        private char[] AsteriskSlashBackQue = new char[] { '*', '/', '\\', '?' };
        private char[] AsteriskQue = new char[] { '*', '?' };
        private char[] FolderSeps = new char[] { '\\', '/' };

        /// <summary>
        /// Creates a new GlobFilter, containing both include and exclude patterns.
        /// When "include" is null / empty, all files are included by default, unless
        /// they match a pattern in "exclude" list.
        /// When "exclude" is null, it is ignored.
        /// </summary>
        /// <param name="include">List of include patterns</param>
        /// <param name="exclude">List of exclude patterns</param>
        public GlobFilter(IEnumerable<string> include, IEnumerable<string> exclude)
            : this(include)
        {
            if (exclude != null)
            {
                var excludes = exclude.Where(x => !string.IsNullOrEmpty(x));
                if (excludes.Any())
                    this.excludeFilter = new GlobFilter(excludes);
            }
        }

        /// <summary>
        /// Creates a new GlobFilter, with just "include" globs.
        /// </summary>
        /// <param name="globs">List of patterns</param>
        public GlobFilter(IEnumerable<string> globs)
        {
            this.matchers = new List<Func<string, bool>>();

            if (globs == null)
                return;

            foreach (var glob in globs)
            {
                if (string.IsNullOrEmpty(glob))
                    continue;

                isActive = true;

                var sep = System.IO.Path.DirectorySeparatorChar;

                var s = glob;
                if (sep == '\\')
                    s = glob.Replace('/', '\\');

                var starDotIndex = s.IndexOf("*.");

                // exact match
                if (s[0] == sep &&
                    s[s.Length - 1] != '/' &&
                    s[s.Length - 1] != '\\' &&
                    s.IndexOfAny(AsteriskQue, 1) < 0)
                {
                    if (exactMatch == null)
                        exactMatch = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

                    exactMatch.Add(s.Substring(1));
                    continue;
                }

                // simple extension filter (*.txt)
                if (starDotIndex == 0 &&
                    s.IndexOfAny(DotAsteriskSlashBackQue, 2) < 0)
                {
                    if (extensions == null)
                        extensions = new HashSet<string>();

                    extensions.Add(s.Substring(1));
                    continue;
                }

                // simple extension filter with extra dot (*.csproj.user)
                if (starDotIndex == 0 &&
                    s.IndexOfAny(AsteriskSlashBackQue, 2) < 0)
                {
                    if (endsWith == null)
                        endsWith = new List<string>();

                    endsWith.Add(s.Substring(1));
                    continue;
                }

                // subdirectory under root and extension (/bin/some/*.txt)
                if (starDotIndex > 1 &&
                    s[starDotIndex - 1] == sep &&
                    s[0] == sep &&
                    s.IndexOfAny(AsteriskSlashBackQue, starDotIndex + 2) < 0 &&
                    s.LastIndexOfAny(AsteriskQue, starDotIndex - 1) < 0)
                {
                    if (startsWithAndEndsWith == null)
                        startsWithAndEndsWith = new List<Tuple<string, bool, string>>();

                    startsWithAndEndsWith.Add(new Tuple<string, bool, string>(
                        s.Substring(1, starDotIndex - 1), false, s.Substring(starDotIndex + 1)));

                    continue;
                }

                // extension under root (/*.txt)
                if (starDotIndex == 1 &&
                    s[0] == sep &&
                    s.IndexOfAny(AsteriskSlashBackQue, starDotIndex + 2) < 0)
                {
                    if (startsWithAndEndsWith == null)
                        startsWithAndEndsWith = new List<Tuple<string, bool, string>>();

                    startsWithAndEndsWith.Add(new Tuple<string, bool, string>(null, false, s.Substring(starDotIndex + 1)));
                    continue;
                }

                // directory filter at any depth (.git/)
                if (s[s.Length - 1] == sep &&
                    s[0] != sep &&
                    s.IndexOfAny(AsteriskQue, 1) < 0)
                {
                    if (contains == null)
                        contains = new List<string>();

                    if (startsWith == null)
                        startsWith = new List<string>();

                    contains.Add(sep.ToString() + s);
                    startsWith.Add(s);

                    continue;
                }

                // directory filter at root (/Imports/)
                if (s[s.Length - 1] == sep &&
                    s[0] == sep &&
                    s.IndexOfAny(AsteriskQue) < 0)
                {
                    if (startsWith == null)
                        startsWith = new List<string>();

                    startsWith.Add(s.Substring(1));
                    continue;
                }

                // folder than any then extension (App_Data/**/*.log) or (/App_Code/**/*.xyz)
                if (starDotIndex > 4 &&
                    s[starDotIndex - 1] == sep &&
                    s[starDotIndex - 2] == '*' &&
                    s[starDotIndex - 3] == '*' &&
                    s[starDotIndex - 4] == sep &&
                    s.LastIndexOfAny(AsteriskQue, starDotIndex - 5) < 0 &&
                    s.IndexOfAny(AsteriskSlashBackQue, starDotIndex + 2) < 0)
                {
                    if (startsWithAndEndsWith == null)
                        startsWithAndEndsWith = new List<Tuple<string, bool, string>>();

                    if (s[0] == sep)
                    {
                        startsWithAndEndsWith.Add(new Tuple<string, bool, string>(
                            s.Substring(1, starDotIndex - 4), true, s.Substring(starDotIndex + 1)));
                    }
                    else
                    {
{'fraction_non_alphanumeric': 0.11846001974333663, 'fraction_numerical': 0.007051191651389085, 'mean_word_length': 1.2505553792446842, 'pattern_counts': {'":': 0, '<': 108, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6789840', 'n_tokens_mistral': 3895, 'n_tokens_neox': 3565, 'n_words': 858}