<commit_before>from __future__ import print_function

import sys
import io
import re
import itertools
import textwrap
from pathlib import Path

try:
    script_name = __file__
except NameError:
    script_name = sys.argv[0]
here = Path(script_name).resolve().parent


sys.path.insert(0, str(here/'pycparser'))


import pycparser, pycparser.c_generator, pycparser.c_ast, pycparser.plyparser


def err(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)


def debug_source(path, line_start, line_end=None):
    with io.open(str(path)) as f:
        lines = list(enumerate(f, 1))
    if line_end is None:
        line_end = line_start+4
        real_line_end = line_start
    else:
        real_line_end = line_end
    return ''.join(
        str(i) + (':' if line_start <= i <= real_line_end else ' ') + '\t' + l
        for i, l in lines[line_start-1-1:line_end+1]
    ).strip('\n')

def debug_source_ast(ast):
    lines = []
    class Visitor(pycparser.c_ast.NodeVisitor):
        def visit(self, node):
            if node.coord and node.coord.line:
                lines.append(node.coord.line)
            self.generic_visit(node)
    Visitor().visit(ast)
    return debug_source(ast.coord.file, min(lines), max(lines))


def parse_c(src):
    try:
        return pycparser.CParser().parse(src)
    except pycparser.plyparser.ParseError as e:
        exc = e
    try:
        m = re.search(r'^(.+?):([0-9]+):[0-9]+:', str(exc))
        err(debug_source(m.group(1), int(m.group(2))))
    except Exception:
        pass
    raise pycparser.plyparser.ParseError(str(exc))


def internal(path):
    if path == '':
        return True<commit_msg>Fix internal detection<commit_after>from __future__ import print_function

import sys
import io
import re
import itertools
import textwrap
from pathlib import Path

try:
    script_name = __file__
except NameError:
    script_name = sys.argv[0]
here = Path(script_name).resolve().parent


sys.path.insert(0, str(here/'pycparser'))


import pycparser, pycparser.c_generator, pycparser.c_ast, pycparser.plyparser


def err(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)


def debug_source(path, line_start, line_end=None):
    with io.open(str(path)) as f:
        lines = list(enumerate(f, 1))
    if line_end is None:
        line_end = line_start+4
        real_line_end = line_start
    else:
        real_line_end = line_end
    return ''.join(
        str(i) + (':' if line_start <= i <= real_line_end else ' ') + '\t' + l
        for i, l in lines[line_start-1-1:line_end+1]
    ).strip('\n')

def debug_source_ast(ast):
    lines = []
    class Visitor(pycparser.c_ast.NodeVisitor):
        def visit(self, node):
            if node.coord and node.coord.line:
                lines.append(node.coord.line)
            self.generic_visit(node)
    Visitor().visit(ast)
    return debug_source(ast.coord.file, min(lines), max(lines))


def parse_c(src):
    try:
        return pycparser.CParser().parse(src)
    except pycparser.plyparser.ParseError as e:
        exc = e
    try:
        m = re.search(r'^(.+?):([0-9]+):[0-9]+:', str(exc))
        err(debug_source(m.group(1), int(m.group(2))))
    except Exception:
        pass
    raise pycparser.plyparser.ParseError(str(exc))


def internal(path):
    if path in ['', '<built-in>']:
        return True
{'fraction_non_alphanumeric': 0.13561356135613561, 'fraction_numerical': 0.007800780078007801, 'mean_word_length': 2.973778307508939, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4271217', 'n_tokens_mistral': 1164, 'n_tokens_neox': 1144, 'n_words': 253}