<commit_before>﻿using System;
using System.IO;
using System.Text.RegularExpressions;

namespace Tabster
{
    public static class Common
    {
        private static readonly Regex NewlineRegex = new Regex("(?<!\r)\n", RegexOptions.Compiled);

        public static string TruncateVersion(string version)
        {
            while (version.EndsWith("0") || version.EndsWith("."))
            {
                version = version.Remove(version.Length - 1, 1);
            }

            return version;
        }

        public static string ConvertNewlines(string source)
        {
            return NewlineRegex.Replace(source, Environment.NewLine);
        }

        public static string StripHTML(string source)
        {
            var array = new char[source.Length];
            var arrayIndex = 0;
            var inside = false;

            for (var i = 0; i < source.Length; i++)
            {
                var let = source[i];
<commit_msg>Added optional enforceDecimal parameter.<commit_after>﻿using System;
using System.IO;
using System.Text.RegularExpressions;

namespace Tabster
{
    public static class Common
    {
        private static readonly Regex NewlineRegex = new Regex("(?<!\r)\n", RegexOptions.Compiled);

        public static string TruncateVersion(string version, bool enforceDecimal = true)
        {
            while (version.EndsWith("0") || version.EndsWith("."))
            {
                version = version.Remove(version.Length - 1, 1);
            }

            if (enforceDecimal && !version.Contains("."))
            {
                version = string.Format("{0}.0", version);
            }

            return version;
        }

        public static string ConvertNewlines(string source)
        {
            return NewlineRegex.Replace(source, Environment.NewLine);
        }

        public static string StripHTML(string source)
        {
            var array = new char[source.Length];
            var arrayIndex = 0;
            var inside = false;

            for (var i = 0; i < source.Length; i++)
            {
                var let = source[i];

{'fraction_non_alphanumeric': 0.10051911278905144, 'fraction_numerical': 0.005663048607833884, 'mean_word_length': 1.7968337730870712, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6153888', 'n_tokens_mistral': 563, 'n_tokens_neox': 536, 'n_words': 146}