<commit_before>using System;
using System.Text;
using System.Collections.Generic;
using System.Linq;

namespace Localization.JsonLocalizer.StringLocalizer
{
    internal static class LocalizerUtil
    {
        internal static string TrimPrefix(string name, string prefix)
        {
            if (name.StartsWith(prefix, StringComparison.Ordinal))
            {
                return name.Substring(prefix.Length);
            }
            return name;
        }

        internal static IEnumerable<string> ExpandPaths(string name, string baseName)
        {
            if (name == null)
            {
                throw new ArgumentNullException(nameof(name));
            }
            return ExpandPathIterator(name, baseName);
        }

        private static IEnumerable<string> ExpandPathIterator(string name, string baseName)
        {
            StringBuilder expansion = new StringBuilder();

            // Start replacing periods, starting at the beginning.
            var components = name.Split(new[] { '.' }, StringSplitOptions.None);
            for (var i = 0; i < components.Length; i++)
            {
                for (var j = 0; j < components.Length; j++)
                {
                    expansion.Append(components[j]).Append(j < i ? '\\' : '.');
                }
                yield return expansion.ToString();
                expansion.Clear();
            }

            // Do the same with the name where baseName prefix is removed.
            var nameWithoutPrefix = TrimPrefix(name, baseName).Substring(1);
            var componentsWithoutPrefix = nameWithoutPrefix.Split(new[] { '.' }, StringSplitOptions.None);
            for (var i = 0; i < componentsWithoutPrefix.Length; i++)
            {
                for (var j = 0; j < componentsWithoutPrefix.Length; j++)
                {
                    expansion.Append(componentsWithoutPrefix[j]).Append(j < i ? '\\' : '.');
                }
                yield return expansion.ToString();
                expansion.Clear();
            }
        }
    }
}<commit_msg>Use platform-independent directory separator char<commit_after>using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;

namespace Localization.JsonLocalizer.StringLocalizer
{
    internal static class LocalizerUtil
    {
        internal static string TrimPrefix(string name, string prefix)
        {
            if (name.StartsWith(prefix, StringComparison.Ordinal))
            {
                return name.Substring(prefix.Length);
            }
            return name;
        }

        internal static IEnumerable<string> ExpandPaths(string name, string baseName)
        {
            if (name == null)
            {
                throw new ArgumentNullException(nameof(name));
            }
            return ExpandPathIterator(name, baseName);
        }

        private static IEnumerable<string> ExpandPathIterator(string name, string baseName)
        {
            StringBuilder expansion = new StringBuilder();

            // Start replacing periods, starting at the beginning.
            var components = name.Split(new[] { '.' }, StringSplitOptions.None);
            for (var i = 0; i < components.Length; i++)
            {
                for (var j = 0; j < components.Length; j++)
                {
                    expansion.Append(components[j]).Append(j < i ? Path.DirectorySeparatorChar : '.');
                }
                yield return expansion.ToString();
                expansion.Clear();
            }

            // Do the same with the name where baseName prefix is removed.
            var nameWithoutPrefix = TrimPrefix(name, baseName).Substring(1);
            var componentsWithoutPrefix = nameWithoutPrefix.Split(new[] { '.' }, StringSplitOptions.None);
            for (var i = 0; i < componentsWithoutPrefix.Length; i++)
            {
                for (var j = 0; j < componentsWithoutPrefix.Length; j++)
                {
                    expansion.Append(componentsWithoutPrefix[j]).Append(j < i ? Path.DirectorySeparatorChar : '.');
                }
                yield return expansion.ToString();
                expansion.Clear();
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.09833372447782211, 'fraction_numerical': 0.002346866932644919, 'mean_word_length': 1.7233226837060702, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4232430', 'n_tokens_mistral': 1041, 'n_tokens_neox': 1013, 'n_words': 258}