<commit_before>#!/usr/bin/env python

# Build the gh-pages

from collections import OrderedDict
import re
import sys
import json

from lintlib import parse_all, log

lint_subheadline = re.compile(r'''^\*\*([\w\s]+?)[:?.!]?\*\*(.*)''')
rust_code_block = re.compile(r'''```rust.+?```''', flags=re.DOTALL)

CONF_TEMPLATE = """\
This lint has the following configuration variables:

* `%s: %s`: %s (defaults to `%s`)."""


def parse_code_block(match):
    lines = []

    for line in match.group(0).split('\n'):
        if not line.startswith('# '):
            lines.append(line)

    return '\n'.join(lines)<commit_msg>Fix some website syntax highlighting<commit_after>#!/usr/bin/env python

# Build the gh-pages

from collections import OrderedDict
import re
import sys
import json

from lintlib import parse_all, log

lint_subheadline = re.compile(r'''^\*\*([\w\s]+?)[:?.!]?\*\*(.*)''')
rust_code_block = re.compile(r'''```rust.+?```''', flags=re.DOTALL)

CONF_TEMPLATE = """\
This lint has the following configuration variables:

* `%s: %s`: %s (defaults to `%s`)."""


def parse_code_block(match):
    lines = []

    for line in match.group(0).split('\n'):
        # fix syntax highlighting for headers like ```rust,ignore
        if line.startswith('```rust'):
            lines.append('```rust')
        elif not line.startswith('# '):
            lines.append(line)

    return '\n'.join(lines)
{'fraction_non_alphanumeric': 0.1957142857142857, 'fraction_numerical': 0.0014285714285714286, 'mean_word_length': 3.67, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7609596', 'n_tokens_mistral': 497, 'n_tokens_neox': 472, 'n_words': 117}