<commit_before># -*- coding: ISO-8859-1 -*-
#-----------------------------------------------------------------------------
# Copyright (c) 2014, HFTools Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
from __future__ import print_function
import io
import itertools
import re

from itertools import chain
from collections import namedtuple

import numpy as np

import hftools.py3compat as py3
from hftools.constants import unit_to_multiplier
from hftools.dataset import DataBlock, DimPartial, hfarray, make_matrix
from hftools.file_formats.common import normalize_names
from hftools.file_formats import merge_blocks
from hftools.utils import glob
from hftools import path

reg_outerfun = re.compile(r"^([A-Za-z_][A-Za-z_0-9]*)\((.*)\)(.*)")


class FileFormatError(Exception):
    pass


class ParseError(Exception):
    pass

Token = namedtuple("Token", 'tag lineno rad')


class Stream(object):
    def __init__(self, stream):
        self.stream = chain(stream)

    def __iter__(self):
        return self

    def __next__(self):
        return next(self.stream)

    def next(self):
        return next(self.stream)

    def push(self, token):
        self.stream = chain([token], self.stream)
<commit_msg>Fix regexp for function matching in variable name<commit_after># -*- coding: ISO-8859-1 -*-
#-----------------------------------------------------------------------------
# Copyright (c) 2014, HFTools Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
from __future__ import print_function
import io
import itertools
import re

from itertools import chain
from collections import namedtuple

import numpy as np

import hftools.py3compat as py3
from hftools.constants import unit_to_multiplier
from hftools.dataset import DataBlock, DimPartial, hfarray, make_matrix
from hftools.file_formats.common import normalize_names
from hftools.file_formats import merge_blocks
from hftools.utils import glob
from hftools import path

reg_outerfun = re.compile(r"^([A-Za-z_][A-Za-z_0-9]*) *\((.*)\)(.*)")


class FileFormatError(Exception):
    pass


class ParseError(Exception):
    pass

Token = namedtuple("Token", 'tag lineno rad')


class Stream(object):
    def __init__(self, stream):
        self.stream = chain(stream)

    def __iter__(self):
        return self

    def __next__(self):
        return next(self.stream)

    def next(self):
        return next(self.stream)

    def push(self, token):
        self.stream = chain([token], self.stream)

{'fraction_non_alphanumeric': 0.19374120956399438, 'fraction_numerical': 0.009142053445850914, 'mean_word_length': 2.4907975460122698, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4490375', 'n_tokens_mistral': 796, 'n_tokens_neox': 776, 'n_words': 249}