<commit_before>#!/usr/bin/env python3
# -*- coding: ascii -*-

import sys, os, re

RAW_IDENT = r'[a-zA-Z_$][a-zA-Z0-9_$]*'
REGEXES = {
    # Identifier. Not matching the "e" in float literals.
    # FIXME: Unicode support.
    'identifier': re.compile(r'(?!<[a-zA-Z0-9_$])' + RAW_IDENT),
    # Interesting statement.
    'import': re.compile(r'(?!<[a-zA-Z0-9_$])import\s+(static\s+)?'
        r'(?P<name>%(i)s\s*(\.\s*%(i)s\s*)*);' % {'i': RAW_IDENT}),
    # Character or string. (The code is supposed to be syntactically valid.)
    'charstring': re.compile(r'''(?s)'([^']|\.)+'|"([^"]|\.)*"'''),
    # Comments.
    'linecomment': re.compile(r'(?m)//.*?$'),
    'blockcomment': re.compile(r'(?s)/\*.*?\*/'),
    # Line separator.
    'newline': re.compile(r'\n')
    }

def regex_race(data, regexes):
    """
    Regex race algorithm, as seen in the Instant frontend
    """
    pos, hits = 0, dict((k, None) for k in regexes)
    while pos != len(data):
        # Update regex matches.
        for k, v in hits.items():
            if v is None or v.start() < pos:
                m = REGEXES[k].search(data, pos)
                if m:
                    hits[k] = m
                else:
                    del hits[k]
        # Abort if no more matches.
        if not hits: break
        # Find first match.
        first = min(hits.values(), key=lambda m: m.start())
        # Emit it.
        yield first
        # Prepare for next round.
        pos = first.end()

def main():
    """
    Main function
    """
    pass

if __name__ == '__main__': main()<commit_msg>[Makefile] [Linter] Correct various issues<commit_after>#!/usr/bin/env python3
# -*- coding: ascii -*-

import sys, os, re

RAW_IDENT = r'[a-zA-Z_$][a-zA-Z0-9_$]*'
KEYWORDS = ('abstract|assert|boolean|break|byte|case|catch|char|class|const|'
    'continue|default|do|double|else|enum|extends|final|finally|float|for|'
    'goto|if|implements|import|instanceof|int|interface|long|native|new|'
    'package|private|protected|public|return|short|static|strictfp|super|'
    'switch|synchronized|this|throw|throws|transient|try|void|volatile|'
    'while')
REGEXES = {
    # Identifier. Not matching the "e" in float literals.
    # FIXME: Unicode support.
    'identifier': re.compile(r'(?!%s)(?<![a-zA-Z0-9_$])%s' %
                             (KEYWORDS, RAW_IDENT)),
    # Interesting statement.
    'import': re.compile(r'(?<![a-zA-Z0-9_$])import\s+(static\s+)?'
        r'(?P<name>%(i)s\s*(\.\s*%(i)s\s*)*);' % {'i': RAW_IDENT}),
    # Character or string. (The code is supposed to be syntactically valid.)
    'charstring': re.compile(r'''(?s)'([^']|\.)+'|"([^"]|\.)*"'''),
    # Comments.
    'linecomment': re.compile(r'(?m)//.*?$'),
    'blockcomment': re.compile(r'(?s)/\*.*?\*/'),
    # Line separator.
    'newline': re.compile(r'\n')
    }

def regex_race(data, regexes):
    """
    Regex race algorithm, as seen in the Instant frontend
    """
    pos, hits = 0, dict((k, None) for k in regexes)
    while pos != len(data):
        # Update regex matches.
        for k, v in tuple(hits.items()):
            if v is None or v.start() < pos:
                m = REGEXES[k].search(data, pos)
                if m:
                    hits[k] = m
                else:
                    del hits[k]
        # Abort if no more matches.
        if not hits: break
        # Find first match.
        first = min(hits.values(), key=lambda m: m.start())
        # Emit preceding data and the match.
        if first.start() != pos: yield data[pos:first.start()]
        yield first
        # Prepare for next round.
        pos = first.end()
    # Emit remainder.
    if pos != len(data):
        yield data[pos:]

def main():
    """
    Main function
    """
    pass

if __name__ == '__main__': main()
{'fraction_non_alphanumeric': 0.19239904988123516, 'fraction_numerical': 0.00422275006598047, 'mean_word_length': 2.3333333333333335, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1030329', 'n_tokens_mistral': 1377, 'n_tokens_neox': 1331, 'n_words': 305}