<commit_before>(function (){
"use strict";

var utils = require("./utils.js"),

STREX = {
 WS : "[ \\t\\u00a0\\u1680\\u180e\\u2000-\\u200a\\u202f\\u205f\\u3000]",
 EOL : "(?=[\\v\\f\\n\u0085\u2028\u2029]|\\r\\n?|$)"
},

RULES = (function (){
return [
 Rule("ESCAPE"   , "\\\\[\\S\\s]"),
 Rule("TH"       , "!!" + STREX.WS),
 Rule("TD"       , "\\|\\|" + STREX.WS),
 Rule("TRSEP"    , "\\|[=]+" + STREX.EOL),
 Rule("ATX_END"  , "=+" + STREX.EOL),
 Rule("ATX"      , "=+"),
 Rule("HR"       , "---[\\-]+" + STREX.EOL),
 Rule("COMMENT"  , "///[/]+" + STREX.EOL),
 Rule("CLASS"    , "\\.\\." + STREX.WS),
 Rule("ID"       , "\\." + STREX.WS),
 Rule("GT"       , ">"),
 Rule("REF"      , ":{"),
 Rule("REF_END"  , "}:"),
 Rule("DD"       , ":" + STREX.WS),
 Rule("DT"       , ";" + STREX.WS),
 Rule("OL"       , "[0-9]+\\." + STREX.WS),
 Rule("OL"       , "#\\." + STREX.WS),
 Rule("DIV"      , "\\*\\*\\*[*]+" + STREX.EOL),
 Rule("UL"       , "[\\-\\+\\*\\u2022\\u2043]" + STREX.WS),
 Rule("PRE"      , "\\\"\\\"[\\\"]+" + STREX.EOL),
 Rule("CODE"     , "\\\"\\\"[\\\"]+"),
 Rule("DEL"      , "{--"),
 Rule("DEL_END"  , "--}"),
 Rule("BOLD"     , "\\*\\*"),
 Rule("INS"      , "\\{\\+\\+"),
 Rule("INS_END"  , "\\+\\+\\}"),
 Rule("SUP"      , "\\^\\^"),
 Rule("SUB"      , ",,"),
 Rule("UNDER"    , "__"),
 Rule("EM"       , "''"),
 Rule("LINK_EXT" , "\\?<"),
 Rule("LINK_IMG" , "!<"),
 Rule("LINK_WIKI", "#<"),
 Rule("LINK_INT" , "#\\["),
 Rule("LINK_CONT", "\\-\\["),
 Rule("BRACKET_R", "\\]"),
 Rule("NL"       , "[\\v\\f\\n]|\\r\\n?"),
 Rule("WS"       , STREX.WS + "+")
];
}()),

ENUM = RULES.reduce(reduceRulesTypes, {TEXT : "TEXT"}),
RENL = /[\v\f\n]|\r\n?/,
REGEX = new RegExp(RULES.map(mapRules).join("|"), "g");



function Rule(name, pattern)<commit_msg>More Precise Line Break Rule<commit_after>(function (){
"use strict";

var utils = require("./utils.js"),

STREX = {
 WS : "[ \\t\\u00a0\\u1680\\u180e\\u2000-\\u200a\\u202f\\u205f\\u3000]",
 EOL : "(?=[\\v\\f\\n\u0085\u2028\u2029]|\\r\\n?|$)"
},

RULES = (function (){
return [
 Rule("ESCAPE"   , "\\\\[\\S\\s]"),
 Rule("TH"       , "!!" + STREX.WS),
 Rule("TD"       , "\\|\\|" + STREX.WS),
 Rule("TRSEP"    , "\\|[=]+" + STREX.EOL),
 Rule("ATX_END"  , "=+" + STREX.EOL),
 Rule("ATX"      , "=+"),
 Rule("HR"       , "---[\\-]+" + STREX.EOL),
 Rule("COMMENT"  , "///[/]+" + STREX.EOL),
 Rule("CLASS"    , "\\.\\." + STREX.WS),
 Rule("ID"       , "\\." + STREX.WS),
 Rule("GT"       , ">"),
 Rule("REF"      , ":{"),
 Rule("REF_END"  , "}:"),
 Rule("DD"       , ":" + STREX.WS),
 Rule("DT"       , ";" + STREX.WS),
 Rule("OL"       , "[0-9]+\\." + STREX.WS),
 Rule("OL"       , "#\\." + STREX.WS),
 Rule("DIV"      , "\\*\\*\\*[*]+" + STREX.EOL),
 Rule("UL"       , "[\\-\\+\\*\\u2022\\u2043]" + STREX.WS),
 Rule("PRE"      , "\\\"\\\"[\\\"]+" + STREX.EOL),
 Rule("CODE"     , "\\\"\\\"[\\\"]+"),
 Rule("DEL"      , "{--"),
 Rule("DEL_END"  , "--}"),
 Rule("BOLD"     , "\\*\\*"),
 Rule("INS"      , "\\{\\+\\+"),
 Rule("INS_END"  , "\\+\\+\\}"),
 Rule("SUP"      , "\\^\\^"),
 Rule("SUB"      , ",,"),
 Rule("UNDER"    , "__"),
 Rule("EM"       , "''"),
 Rule("LINK_EXT" , "\\?<"),
 Rule("LINK_IMG" , "!<"),
 Rule("LINK_WIKI", "#<"),
 Rule("LINK_INT" , "#\\["),
 Rule("LINK_CONT", "\\-\\["),
 Rule("BRACKET_R", "\\]"),
 Rule("NL"       , "[\\v\\f\\n\u0085\u2028\u2029]|\\r\\n?"),
 Rule("WS"       , STREX.WS + "+")
];
}()),

ENUM = RULES.reduce(reduceRulesTypes, {TEXT : "TEXT"}),
RENL = new RegExp(/[\v\f\n\u0085\u2028\u2029]|\r\n?/),
REGEX = new RegExp(RULES.map(mapRules).join("|"), "g");



function Rule(name, pattern)
{'fraction_non_alphanumeric': 0.37656903765690375, 'fraction_numerical': 0.03403068340306834, 'mean_word_length': 2.1903914590747333, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2427948', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1630, 'n_words': 174}