<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;

namespace QualityEnforcer
{
    public class QualityRules
    {
        public QualityRules()
        {
            // Default rules
            TrimTrailingLines = true;
            TrimTrailingWhitespace = true;
            LineEndings = LineEndingStyle.Detect;
            Indentation = IndentationStyle.Detect;
            NumberOfSpaces = null;
        }

        /// <summary>
        /// If DetectLineEndings is false, this value is used to determine line ending style.
        /// </summary>
        public LineEndingStyle LineEndings { get; set; }
        /// <summary>
        /// If DetectIndentation is false, this value is used to determine indent style.
        /// </summary>
        public IndentationStyle Indentation { get; set; }
        /// <summary>
        /// If Indentation is set to use spaces, this number determines how many to use.
        /// </summary>
        public int? NumberOfSpaces { get; set; }
        /// <summary>
        /// If true, excess empty lines at the end of a file are removed.
        /// </summary>
        public bool TrimTrailingLines { get; set; }
        /// <summary>
        /// If true, excess tabs and spaces are trimmed from the end of each line.
        /// </summary>
        public bool TrimTrailingWhitespace { get; set; }

        public static QualityRules FromFile(string file)
        {
            QualityRules rules = new QualityRules();
            var lines = File.ReadAllLines(file);
            foreach (var line in lines)
            {
                if (line.StartsWith("* "))
                {
                    // Attempt to interpret as a rule
                    string key = line.Substring(2);
                    string value = null;
                    if (key.Contains(":"))
                    {
                        value = key.Substring(key.IndexOf(":") + 1).Trim().ToLower();
                        key = key.Remove(key.IndexOf(":"));
                    }
                    key = key.Trim().ToLower();
                    switch (key)
                    {
                        case "line endings":
                            if (value == null) break;
                            if (value == "crlf") rules.LineEndings = LineEndingStyle.CRLF;
                            if (value == "lf") rules.LineEndings = LineEndingStyle.LF;
                            break;
                        case "indentation":
                            if (value == null) break;
                            if (value == "tabs") rules.Indentation = IndentationStyle.Tabs;
                            int spaces;
                            if (value.EndsWith("spaces") && int.TryParse(
                                value.Remove(value.IndexOf(' ')), out spaces))
                            {
                                rules.Indentation = IndentationStyle.Spaces;
                                rules.NumberOfSpaces = spaces;
                            }
                            break;
                        case "trim trailing lines":
                            rules.TrimTrailingLines = true;
                            break;
                        case "trim trailing whitespace":
                            rules.TrimTrailingWhitespace = true;
                            break;
                    }
                }
            }
            return rules;
        }
    }
}<commit_msg>Updated QualityRules<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;

namespace QualityEnforcer
{
    public class QualityRules
    {
        public QualityRules()
        {
            // Default rules
            TrimTrailingLines = true;
            TrimTrailingWhitespace = true;
            LineEndings = LineEndingStyle.Detect;
            Indentation = IndentationStyle.Detect;
            NumberOfSpaces = null;
        }

        /// <summary>
        /// If DetectLineEndings is false, this value is used to determine line ending style.
        /// </summary>
        public LineEndingStyle LineEndings { get; set; }
        /// <summary>
        /// If DetectIndentation is false, this value is used to determine indent style.
        /// </summary>
        public IndentationStyle Indentation { get; set; }
        /// <summary>
        /// If Indentation is set to use spaces, this number determines how many to use.
        /// </summary>
        public int? NumberOfSpaces { get; set; }
        /// <summary>
        /// If true, excess empty lines at the end of a file are removed.
        /// </summary>
        public bool TrimTrailingLines { get; set; }
        /// <summary>
        /// If true, excess tabs and spaces are trimmed from the end of each line.
        /// </summary>
        public bool TrimTrailingWhitespace { get; set; }

        public static QualityRules Parse(string contents)
        {
            QualityRules rules = new QualityRules();
            var lines = contents.Replace("\r", "").Split('\n');
            foreach (var line in lines)
            {
                if (line.StartsWith("* "))
                {
                    // Attempt to interpret as a rule
                    string key = line.Substring(2);
                    string value = null;
                    if (key.Contains(":"))
                    {
                        value = key.Substring(key.IndexOf(":") + 1).Trim().ToLower();
                        key = key.Remove(key.IndexOf(":"));
                    }
                    key = key.Trim().ToLower();
                    switch (key)
                    {
                        case "line endings":
                            if (value == null) break;
                            if (value == "crlf") rules.LineEndings = LineEndingStyle.CRLF;
                            if (value == "lf") rules.LineEndings = LineEndingStyle.LF;
                            break;
                        case "indentation":
                            if (value == null) break;
                            if (value == "tabs") rules.Indentation = IndentationStyle.Tabs;
                            int spaces;
                            if (value.EndsWith("spaces") && int.TryParse(
                                value.Remove(value.IndexOf(' ')), out spaces))
                            {
                                rules.Indentation = IndentationStyle.Spaces;
                                rules.NumberOfSpaces = spaces;
                            }
                            break;
                        case "trim trailing lines":
                            rules.TrimTrailingLines = true;
                            break;
                        case "trim trailing whitespace":
                            rules.TrimTrailingWhitespace = true;
                            break;
                    }
                }
            }
            return rules;
        }

        public static QualityRules FromFile(string file)
        {
            var text = File.ReadAllText(file);
            return QualityRules.Parse(text);
        }
    }
}
{'fraction_non_alphanumeric': 0.0914119762135251, 'fraction_numerical': 0.0005531738348776103, 'mean_word_length': 1.1208211143695015, 'pattern_counts': {'":': 14, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6695182', 'n_tokens_mistral': 1742, 'n_tokens_neox': 1662, 'n_words': 472}