<commit_before><commit_msg>Base changelog lexer<commit_after>from pygments.lexer import RegexLexer, bygroups
from pygments.token import *

DATE_RE = r'\d\d [A-Z][a-z]{2} \d{4}'
EMAIL_RE = r'[\w\.\-]+@(?:\w+\.)+\w+'

# Literal.Date

class ChangelogLexer(RegexLexer):
    name = 'Changelog'
    aliases = ['changelog']
    filenames = ['Changelog']

    tokens = {
        'root': [
            (r'^# .*\n', Comment.Single), # Comment
            (r'^(\*)(.+)( )(\()(%s)(\))' % DATE_RE, bygroups(Operator, String, Name, Punctuation, Keyword, Punctuation)),
            #(r'^  ' , Whitespace, 'main'),
            (r'^(  )(%(date)s)(;)( +)([^<]*)(<)(%(email)s)(>)' % {'date': DATE_RE, 'email': EMAIL_RE},
                bygroups(Whitespace, Keyword, Punctuation, Whitespace, Name.Variable, Punctuation, Number, Number ), 'main'), # Date
            (EMAIL_RE, Number),
        ],
        'main': [
            (r' *\n\n', Punctuation, '#pop'),
            (DATE_RE, Keyword), # Date
            (EMAIL_RE, Number), # email
            (r'(,| |  )(\+[\w\.\-\+\/]+)', bygroups(Punctuation, Generic.Inserted)),
            (r'(,| |  )(\-[\w\.\-\+\/]+)', bygroups(Punctuation, Generic.Deleted)),
            (r'(,| |  )(\w[\w\.\-\+\/]+)', bygroups(Punctuation, Operator.Word)),
            (r':( +|\n)', Punctuation, 'message'),
            (r';|<|>|,', Punctuation),
            (r' |\n', Whitespace),
        ],
        'message': [
            (r' *\n', Punctuation, '#pop:2'),
            #(r'#\d+', String),
            (r'  ', Whitespace, 'message_text'),
        ],
        'message_text': [
            (r' *\n', Whitespace, '#pop'),
            (r'(?i)bug #\d+', String),
            (r'#\d+', String),
            (r'(?i)bug \d+', String),
            (r'\(|\)', Punctuation),
            (r'[^#\n ]+', Generic),
            (r' +', Whitespace),
        ]
    }
{'fraction_non_alphanumeric': 0.2552278820375335, 'fraction_numerical': 0.0016085790884718498, 'mean_word_length': 1.9951845906902086, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2269954', 'n_tokens_mistral': 706, 'n_tokens_neox': 684, 'n_words': 123}