<commit_before>import os
import sys
import re

from xml.dom import minidom

def parse():
    scriptpath = os.path.realpath(__file__)
    projectpath = os.path.dirname(os.path.dirname(scriptpath))
    projectpath = os.path.join(projectpath, "src/client/intercept/client")
    projectpath_sqf = os.path.join(projectpath, "sqf")

    implementations = []
    declarations = []

    # Walk through our source files and figure out which ones are used in our code
    for file in os.listdir(projectpath_sqf):
        foundInFile = 0
        if file.endswith(".cpp") or file.endswith(".hpp"):
            source_file = os.path.join(projectpath_sqf, file)
            lineN = 0
            with open(source_file) as f:
                filecontents = re.sub("(/\*[.\s\w\W]*?\*/)", '', f.read())
                filecontents = re.sub("(//.*?\n)", '', filecontents)

                if file.endswith(".cpp"):
                    match_impls = re.findall("([a-zA-Z0-9:_<>]+,*?\s*[a-zA-Z0-9:_<>]+>|[a-zA-Z0-9:_<>]+)\s+([a-zA-Z0-9_]+)(\(.*?\))(?=\s*\{)", filecontents)
                    for match_impl in match_impls:
                        if (match_impl):
                            foundInFile += 1
                            if (match_impl[1].startswith("__")):
                                continue
                            implementations.append([match_impl[0], match_impl[1], match_impl[2], [file, lineN]]) # full contract of our implementation
                if file.endswith(".hpp"):
                    match_impls = re.findall("([a-zA-Z0-9:_<>]+,*?\s*[a-zA-Z0-9:_<>]+>|[a-zA-Z0-9:_<>]+)\s+([a-zA-Z0-9_]+)(\(.*?\))(?=\s*\;)", filecontents)
                    for match_impl in match_impls:
                        if (match_impl):
                            foundInFile += 1
                            if (match_impl[1].startswith("__")):
                                continue
                            declarations.append([match_impl[0], match_impl[1], match_impl[2], [file, lineN]]) # full contract of our declaration

    errors_found = []
    warnings_found = []

    parsed_impl = []
    # strip default values
    for impl in implementations:
        impl[2] = re.sub("(\s*=\s*[a-zA-Z0-9\"]+)(?=[,\)])", r"", impl[2]).lower()
        elements = impl[2].split(',')
        if (impl[2] == "()"):
            parsed_impl.append(impl)
            continue
        impl[2] = ''
        to_add = False
        for el in elements:<commit_msg>Fix handling of declarations<commit_after>import os
import sys
import re

from xml.dom import minidom

def parse():
    scriptpath = os.path.realpath(__file__)
    projectpath = os.path.dirname(os.path.dirname(scriptpath))
    projectpath = os.path.join(projectpath, "src/client/intercept/client")
    projectpath_sqf = os.path.join(projectpath, "sqf")

    implementations = []
    declarations = []

    # Walk through our source files and figure out which ones are used in our code
    for file in os.listdir(projectpath_sqf):
        foundInFile = 0
        if file.endswith(".cpp") or file.endswith(".hpp"):
            source_file = os.path.join(projectpath_sqf, file)
            lineN = 0
            with open(source_file) as f:
                filecontents = re.sub("(/\*[.\s\w\W]*?\*/)", '', f.read())
                filecontents = re.sub("(//.*?\n)", '', filecontents)

                #if file.endswith(".cpp"):
                match_impls = re.findall("([a-zA-Z0-9:_<>]+,*?\s*[a-zA-Z0-9:_<>]+>|[a-zA-Z0-9:_<>]+)\s+([a-zA-Z0-9_]+)(\(.*?\))(?=\s*\{)", filecontents)
                for match_impl in match_impls:
                    if (match_impl):
                        foundInFile += 1
                        if (match_impl[1].startswith("__")):
                            continue
                        implementations.append([match_impl[0], match_impl[1], match_impl[2], [file, lineN]]) # full contract of our implementation
                #if file.endswith(".hpp"):
                match_impls = re.findall("([a-zA-Z0-9:_<>]+,*?\s*[a-zA-Z0-9:_<>]+>|[a-zA-Z0-9:_<>]+)\s+([a-zA-Z0-9_]+)(\(.*?\))(?=\s*\;)", filecontents)
                for match_impl in match_impls:
                    if (match_impl):
                        foundInFile += 1
                        if (match_impl[1].startswith("__")):
                            continue
                        # Check if we have actually matched a valid declaration
                        valid_match = re.search("[a-zA-Z0-9:<>_]+\s?[&\*]?\s+[&\*]?[a-zA-Z0-9_]+", match_impl[2])
                        if (valid_match or match_impl[2] == "()"):
                            declarations.append([match_impl[0], match_impl[1], match_impl[2], [file, lineN]]) # full contract of our declaration

    errors_found = []
    warnings_found = []

    parsed_impl = []
    # strip default values
    for impl in implementations:
        impl[2] = re.sub("(\s*=\s*[a-zA-Z0-9\"]+)(?=[,\)])", r"", impl[2]).lower()
        elements = impl[2].split(',')
        if (impl[2] == "()"):
            parsed_impl.append(impl)
            continue
        impl[2] = ''
        to_add = False
        for el in elements:
{'fraction_non_alphanumeric': 0.17626459143968873, 'fraction_numerical': 0.014785992217898832, 'mean_word_length': 1.8325068870523415, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7368530', 'n_tokens_mistral': 1714, 'n_tokens_neox': 1653, 'n_words': 304}