<commit_before>#!/usr/bin/env python
# Generate the wiki Home.md page from the contained doc comments
# requires the checked out wiki in ../rust-clippy.wiki/
# with -c option, print a warning and set exit status 1 if the file would be
# changed.
import os
import re
import sys


level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''')
conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE)
confvar_re = re.compile(r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''')


def parse_path(p="src"):
    d = {}
    for f in os.listdir(p):
        if f.endswith(".rs"):
            parse_file(d, os.path.join(p, f))
    return (d, parse_conf(p))


def parse_conf(p):
    c = {}
    with open(p + '/utils/conf.rs') as f:
        f = f.read()

        m = re.search(conf_re, f)
        m = m.groups()[0]

        m = re.findall(confvar_re, m)

        for (lint, doc, name, default, ty) in m:
            c[lint.lower()] = (name, ty, doc, default)

    return c


def parse_file(d, f):
    last_comment = []
    comment = True

    with open(f) as rs:
        for line in rs:
            if comment:
                if line.startswith("///"):
                    if line.startswith("/// "):
                        last_comment.append(line[4:])
                    else:
                        last_comment.append(line[3:])
                elif line.startswith("declare_lint!"):
                    comment = False
                    deprecated = False
                elif line.startswith("declare_deprecated_lint!"):
                    comment = False
                    deprecated = True
                else:
                    last_comment = []
            if not comment:
                l = line.strip()
                m = re.search(r"pub\s+([A-Z_]+)", l)

                if m:
                    name = m.group(1).lower()

                    # Intentionally either a never looping or infinite loop
                    while not deprecated:
                        m = re.search(level_re, line)
                        if m:
                            level = m.group(0)
                            break

                        line = next(rs)

                    if deprecated:<commit_msg>Fix wiki links and `char_lit_as_u8`<commit_after>#!/usr/bin/env python
# Generate the wiki Home.md page from the contained doc comments
# requires the checked out wiki in ../rust-clippy.wiki/
# with -c option, print a warning and set exit status 1 if the file would be
# changed.
import os
import re
import sys


level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''')
conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE)
confvar_re = re.compile(r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''')


def parse_path(p="src"):
    d = {}
    for f in os.listdir(p):
        if f.endswith(".rs"):
            parse_file(d, os.path.join(p, f))
    return (d, parse_conf(p))


def parse_conf(p):
    c = {}
    with open(p + '/utils/conf.rs') as f:
        f = f.read()

        m = re.search(conf_re, f)
        m = m.groups()[0]

        m = re.findall(confvar_re, m)

        for (lint, doc, name, default, ty) in m:
            c[lint.lower()] = (name, ty, doc, default)

    return c


def parse_file(d, f):
    last_comment = []
    comment = True

    with open(f) as rs:
        for line in rs:
            if comment:
                if line.startswith("///"):
                    if line.startswith("/// "):
                        last_comment.append(line[4:])
                    else:
                        last_comment.append(line[3:])
                elif line.startswith("declare_lint!"):
                    comment = False
                    deprecated = False
                elif line.startswith("declare_deprecated_lint!"):
                    comment = False
                    deprecated = True
                else:
                    last_comment = []
            if not comment:
                l = line.strip()
                m = re.search(r"pub\s+([A-Z_][A-Z_0-9]*)", l)

                if m:
                    name = m.group(1).lower()

                    # Intentionally either a never looping or infinite loop
                    while not deprecated:
                        m = re.search(level_re, line)
                        if m:
                            level = m.group(0)
                            break

                        line = next(rs)

                    if deprecated:
{'fraction_non_alphanumeric': 0.14295227524972254, 'fraction_numerical': 0.003329633740288568, 'mean_word_length': 1.3904509283819628, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3582689', 'n_tokens_mistral': 1428, 'n_tokens_neox': 1339, 'n_words': 349}