<commit_before># Copyrighted, see License.txt for more information.
#-------------------------------------------------------------------------------
# Coonfiguration
#-------------------------------------------------------------------------------

# Everything but the static/componets directory should be listed.  Algorithm is
# recursive, no need to list sub directories if search is meant to be all 
# inclusive.
app_dirs = [
    '~/ipython/IPython/html/auth',
    '~/ipython/IPython/html/base',
    '~/ipython/IPython/html/nbconvert',
    '~/ipython/IPython/html/notebook',
    '~/ipython/IPython/html/services',
    '~/ipython/IPython/html/templates',
    '~/ipython/IPython/html/tests',
    '~/ipython/IPython/html/tree',
    '~/ipython/IPython/html/widgets',
    '~/ipython/IPython/html/static/auth',
    '~/ipython/IPython/html/static/base',
    '~/ipython/IPython/html/static/custom',
    '~/ipython/IPython/html/static/dateformat',
    '~/ipython/IPython/html/static/notebook',
    '~/ipython/IPython/html/static/services',
    '~/ipython/IPython/html/static/tree',
    '~/ipython/IPython/html/static/widgets',
]

#-------------------------------------------------------------------------------
# Do not edit below this line
#-------------------------------------------------------------------------------

import re
import os
import sys

re_html = re.compile('.*\\.html$|.*\\.htm$')
re_js = re.compile('.*\\.js$')
re_less = re.compile('.*\\.less$|.*\\.css$')
re_py = re.compile('.*\\.py$')

app_dirs = [os.path.expanduser(p) for p in app_dirs]
get_files = lambda regex: [os.path.join(dirpath, f)
    for app_dir in app_dirs
    for dirpath, dirnames, files in os.walk(app_dir)
    for f in files if regex.match(f)]
htmlfiles = get_files(re_html)
jsfiles = get_files(re_js)
lessfiles = get_files(re_less)
pyfiles = get_files(re_py)

css_class = [r"([\.@])({0})([ {{\(\.#:])"]
html_class = [
    "(\".*? )({0})( .*?\")", 
    "(\".*? )({0})(\")", 
    "(\")({0})( .*?\")", 
    "(\")({0})(\")", 
    "('.*? )({0})(')", 
    "(')({0})( .*?')", 
    "(')({0})(')", 
    "('.*? )({0})( .*?')", 
]
selector_class = ["([\"].*?\\.)({0})([ \\.#:\"])", "('.*?\\.)({0})([ \\.#:'])"]

quick_replacements = {
    "row-fluid": "row",
    "container-fluid": "container",
    "brand": "navbar-brand",
    "nav-collapse": "navbar-collapse",
    "nav-toggle": "navbar-toggle",
    "btn-navbar": "navbar-btn",
    "hero-unit": "jumbotron",
    "btn-mini": "btn-xs",
    "btn-small": "btn-sm",
    "btn-large": "btn-lg",
    "alert-error": "alert-danger",
    "visible-phone": "visible-xs",
    "visible-tablet": "visible-sm",
    "hidden-phone": "hidden-xs",
    "hidden-tablet": "hidden-sm",
    "input-block-level": "form-control",
    "control-group": "form-group",
    "input-prepend": "input-group",
    "input-append": "input-group",
    "add-on": "input-group-addon",
    "img-polaroid": "img-thumbnail",
    "unstyled": "list-unstyled",
    "inline": "list-inline",
    "muted": "text-muted",
    "label-important": "label-danger",
    "text-error": "text-danger",
    "bar": "progress-bar",
    "accordion": "panel-group",
    "accordion-heading": "panel-heading",
    "accordion-body": "panel-collapse",
    "accordion-inner": "panel-body",
    "checkbox.inline": "checkbox-inline",
    "radio.inline": "radio-inline",
}
replace_split_class = {
    'btn': ['btn', 'btn-default'],
    'label': ['label', 'label-default'],
    'accordion-group': ['panel', 'panel-default'],
}
warn_abouts = {
    'inline': 'If this is a radio or checkbox, replace ".inline" with ".checkbox-inline" or ".radio-inline".',
    'error': 'If this is a ".table.error", replace it with ".table.danger".\n\t".control-group.*" should be replaced with ".form-group.has-*".',
    'bar-.*?': '"bar-*" should be replaced with ".progress-bar-*".',
    'icon-.*?': '"icon-*" should be replaced with ".glyphicon .glyphicon-*".',
    'span.*?': '"span.*" should be replaced with ".col-md-*".',
    'offset.*?': '"offset.*" should be replaced with ".col-md-offset-*".',
    'success': '".control-group.*" should be replaced with ".form-group.has-*".',
    'warning': '".control-group.*" should be replaced with ".form-group.has-*".',
    '.hidden-desktop': 'Split into .hidden-md .hidden-lg',
    '.visible-desktop': 'Split into .visible-md .visible-lg',
    'form-actions': '"Form actions" removed; replaced by "N/A".',
    'form-search': '"Search form" removed; replaced by "N/A".',
    'control-group.info': '"Form group with info" removed; replaced by "N/A".',
    'input-mini': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-small': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-medium': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-large': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-xlarge': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-xxlarge': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-block-level': '"Block level form input" removed; replaced by "No direct equivalent, but forms controlsare similar.".',
    'btn-inverse': '"Inverse buttons" removed; replaced by "N/A".',
    'row-fluid': '"Fluid row" removed; replaced by ".row (no more fixed grid)".',
    'controls': '"Controls wrapper" removed; replaced by "N/A".',
    'controls-row': '"Controls row" removed; replaced by ".row or .form-group".',
    'navbar-inner': '"Navbar inner" removed; replaced by "N/A".',
    'divider-vertical': '"Navbar vertical dividers" removed; replaced by "N/A".',
    'dropdown-submenu': '"Dropdown submenu" removed; replaced by "N/A".',
    'tabs-left': '"Tab alignments" removed; replaced by "N/A".',
    'tabs-right': '"Tab alignments" removed; replaced by "N/A".',
    'tabs-below': '"Tab alignments" removed; replaced by "N/A".',
    'pill-content': '"Pill-based tabbable area" removed; replaced by ".tab-content".',
    'pill-pane': '"Pill-based tabbable area pane" removed; replaced by ".tab-pane".',
    'nav-header': '"Nav lists" removed; replaced by "No direct equivalent, but list groups and.panel-groups are similar.".',
    'help-inline': '"Inline help for form controls" removed; replaced by "No exact equivalent, but .help-block is similar.".',
}

def progress(percent):
    print '\r[' + ('#' * int(percent * 50)) + (' ' * int((1-percent) * 50)) + ']',
    sys.stdout.flush()
def regex_sub_file(filename, regex, repl):
    with open(filename, 'r') as f:
        contents = f.read()
    (contents, count) = regex.subn(repl, contents)
    with open(filename, 'w') as f:<commit_msg>Fixed bug in encoding<commit_after># Copyrighted, see License.txt for more information.
#-------------------------------------------------------------------------------
# Coonfiguration
#-------------------------------------------------------------------------------

# Everything but the static/componets directory should be listed.  Algorithm is
# recursive, no need to list sub directories if search is meant to be all 
# inclusive.
app_dirs = [
    '~/ipython/IPython/html/auth',
    '~/ipython/IPython/html/base',
    '~/ipython/IPython/html/nbconvert',
    '~/ipython/IPython/html/notebook',
    '~/ipython/IPython/html/services',
    '~/ipython/IPython/html/templates',
    '~/ipython/IPython/html/tests',
    '~/ipython/IPython/html/tree',
    '~/ipython/IPython/html/widgets',
    '~/ipython/IPython/html/static/auth',
    '~/ipython/IPython/html/static/base',
    '~/ipython/IPython/html/static/custom',
    '~/ipython/IPython/html/static/dateformat',
    '~/ipython/IPython/html/static/notebook',
    '~/ipython/IPython/html/static/services',
    '~/ipython/IPython/html/static/tree',
    '~/ipython/IPython/html/static/widgets',
]

#-------------------------------------------------------------------------------
# Do not edit below this line
#-------------------------------------------------------------------------------

import re
import os
import sys

re_html = re.compile('.*\\.html$|.*\\.htm$')
re_js = re.compile('.*\\.js$')
re_less = re.compile('.*\\.less$|.*\\.css$')
re_py = re.compile('.*\\.py$')

app_dirs = [os.path.expanduser(p) for p in app_dirs]
get_files = lambda regex: [os.path.join(dirpath, f)
    for app_dir in app_dirs
    for dirpath, dirnames, files in os.walk(app_dir)
    for f in files if regex.match(f)]
htmlfiles = get_files(re_html)
jsfiles = get_files(re_js)
lessfiles = get_files(re_less)
pyfiles = get_files(re_py)

css_class = [r"([\.@])({0})([ {{\(\.#:])"]
html_class = [
    "(\".*? )({0})( .*?\")", 
    "(\".*? )({0})(\")", 
    "(\")({0})( .*?\")", 
    "(\")({0})(\")", 
    "('.*? )({0})(')", 
    "(')({0})( .*?')", 
    "(')({0})(')", 
    "('.*? )({0})( .*?')", 
]
selector_class = ["([\"].*?\\.)({0})([ \\.#:\"])", "('.*?\\.)({0})([ \\.#:'])"]

quick_replacements = {
    "row-fluid": "row",
    "container-fluid": "container",
    "brand": "navbar-brand",
    "nav-collapse": "navbar-collapse",
    "nav-toggle": "navbar-toggle",
    "btn-navbar": "navbar-btn",
    "hero-unit": "jumbotron",
    "btn-mini": "btn-xs",
    "btn-small": "btn-sm",
    "btn-large": "btn-lg",
    "alert-error": "alert-danger",
    "visible-phone": "visible-xs",
    "visible-tablet": "visible-sm",
    "hidden-phone": "hidden-xs",
    "hidden-tablet": "hidden-sm",
    "input-block-level": "form-control",
    "control-group": "form-group",
    "input-prepend": "input-group",
    "input-append": "input-group",
    "add-on": "input-group-addon",
    "img-polaroid": "img-thumbnail",
    "unstyled": "list-unstyled",
    "inline": "list-inline",
    "muted": "text-muted",
    "label-important": "label-danger",
    "text-error": "text-danger",
    "bar": "progress-bar",
    "accordion": "panel-group",
    "accordion-heading": "panel-heading",
    "accordion-body": "panel-collapse",
    "accordion-inner": "panel-body",
    "checkbox.inline": "checkbox-inline",
    "radio.inline": "radio-inline",
}
replace_split_class = {
    'btn': ['btn', 'btn-default'],
    'label': ['label', 'label-default'],
    'accordion-group': ['panel', 'panel-default'],
}
warn_abouts = {
    'inline': 'If this is a radio or checkbox, replace ".inline" with ".checkbox-inline" or ".radio-inline".',
    'error': 'If this is a ".table.error", replace it with ".table.danger".\n\t".control-group.*" should be replaced with ".form-group.has-*".',
    'bar-.*?': '"bar-*" should be replaced with ".progress-bar-*".',
    'icon-.*?': '"icon-*" should be replaced with ".glyphicon .glyphicon-*".',
    'span.*?': '"span.*" should be replaced with ".col-md-*".',
    'offset.*?': '"offset.*" should be replaced with ".col-md-offset-*".',
    'success': '".control-group.*" should be replaced with ".form-group.has-*".',
    'warning': '".control-group.*" should be replaced with ".form-group.has-*".',
    '.hidden-desktop': 'Split into .hidden-md .hidden-lg',
    '.visible-desktop': 'Split into .visible-md .visible-lg',
    'form-actions': '"Form actions" removed; replaced by "N/A".',
    'form-search': '"Search form" removed; replaced by "N/A".',
    'control-group.info': '"Form group with info" removed; replaced by "N/A".',
    'input-mini': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-small': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-medium': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-large': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-xlarge': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-xxlarge': '"Fixed-width input sizes" removed; replaced by "Use .form-control and the grid systeminstead.".',
    'input-block-level': '"Block level form input" removed; replaced by "No direct equivalent, but forms controlsare similar.".',
    'btn-inverse': '"Inverse buttons" removed; replaced by "N/A".',
    'row-fluid': '"Fluid row" removed; replaced by ".row (no more fixed grid)".',
    'controls': '"Controls wrapper" removed; replaced by "N/A".',
    'controls-row': '"Controls row" removed; replaced by ".row or .form-group".',
    'navbar-inner': '"Navbar inner" removed; replaced by "N/A".',
    'divider-vertical': '"Navbar vertical dividers" removed; replaced by "N/A".',
    'dropdown-submenu': '"Dropdown submenu" removed; replaced by "N/A".',
    'tabs-left': '"Tab alignments" removed; replaced by "N/A".',
    'tabs-right': '"Tab alignments" removed; replaced by "N/A".',
    'tabs-below': '"Tab alignments" removed; replaced by "N/A".',
    'pill-content': '"Pill-based tabbable area" removed; replaced by ".tab-content".',
    'pill-pane': '"Pill-based tabbable area pane" removed; replaced by ".tab-pane".',
    'nav-header': '"Nav lists" removed; replaced by "No direct equivalent, but list groups and.panel-groups are similar.".',
    'help-inline': '"Inline help for form controls" removed; replaced by "No exact equivalent, but .help-block is similar.".',
}

def progress(percent):
    print '\r[' + ('#' * int(percent * 50)) + (' ' * int((1-percent) * 50)) + ']',
    sys.stdout.flush()
def regex_sub_file(filename, regex, repl):
    with open(filename, 'r') as f:
        contents = f.read()
    (contents, count) = regex.subn(repl, contents)
    with open(filename, 'w') as f:
{'fraction_non_alphanumeric': 0.2545242265032107, 'fraction_numerical': 0.002335084646818447, 'mean_word_length': 2.8868406125921724, 'pattern_counts': {'":': 66, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '619782', 'n_tokens_mistral': 4735, 'n_tokens_neox': 4568, 'n_words': 1102}