<commit_before># -*- coding: utf-8 -*-

import xml.dom.minidom
import model

def _get_struct_name_and_index(struct_list, struct_dict):
    for x in struct_list:
        index = struct_list.index(x)
        struct_dict[x.name] = index

def _sorter(struct_list, index, out_list, struct_dict, lista):
    element = struct_list[index]
    element_name = element.name
    for member_elem in element.members:
        member_type = member_elem.type
        if member_type.startswith('S') and member_type in struct_dict:
            index = struct_dict[member_type]
            x = struct_list[index]
            if x not in out_list:
                out_list.extend(_sorter(struct_list, index, out_list, struct_dict, lista))
        else:
            for x in struct_list:
                if member_type in x.name:
                    if x not in out_list:
                        out_list.append(x)
    if element not in out_list:
        out_list.append(element)
    return lista

def sort_struct(struct_list):
    index = 0
    out_list = []
    struct_dict = {}
    lista = []
    _get_struct_name_and_index(struct_list, struct_dict)
    for struct_elem_index in xrange(len(struct_list)):
        out_list.extend(_sorter(struct_list, struct_elem_index, out_list, struct_dict, lista))
    return out_list

class IsarParser(object):

    primitive_types = {"8 bit integer unsigned": "u8",
                       "16 bit integer unsigned": "u16",
                       "32 bit integer unsigned": "u32",
                       "64 bit integer unsigned": "u64",
                       "8 bit integer signed": "i8",
                       "16 bit integer signed": "i16",
                       "32 bit integer signed": "i32",
                       "64 bit integer signed": "i64",
                       "32 bit float": "r32",
                       "64 bit float": "r64"}

    def __get_struct_members(self, elem):
        members = []
        kname = elem.attributes["name"].value
        ktype = elem.attributes["type"].value
        if elem.getElementsByTagName('dimension'):
            dimension = dict(elem.getElementsByTagName('dimension')[0].attributes.items())
            if "isVariableSize" in dimension:
                type = dimension.get("variableSizeFieldType", "u32")
                name = dimension.get("variableSizeFieldName", kname + "_len")
                members.append(model.Struct.Member(name, type, None, None, None))
                members.append(model.Struct.Member(kname, ktype, True, name, None))
            elif "size" in dimension:
                size = dimension["size"]
                members.append(model.Struct.Member(kname, ktype, True, None, size))
        else:
            members.append(model.Struct.Member(kname, ktype, None, None, None))
        return members

    def __get_struct(self, elem):
        msg = model.Struct()
        msg.name = elem.attributes["name"].value
        msg.members = reduce(lambda x, y: x + y, (self.__get_struct_members(member)
                                                  for member
                                                  in elem.getElementsByTagName('member')), [])
        return msg

    def __get_structs(self, dom):
        return [self.__get_struct(elem)
                for elem
                in dom.getElementsByTagName("struct") + dom.getElementsByTagName("message")
                if elem.hasChildNodes()]

    def __get_enum_member(self, elem):
        value = elem.getAttribute('value')
        value = value if value != "-1" else "0xFFFFFFFF"
        return (elem.attributes["name"].value, value)

    def __get_enum(self, elem):
        if not elem.hasChildNodes():
            return None
        name = elem.attributes["name"].value
        enumerators = [self.__get_enum_member(member) for member in elem.getElementsByTagName('enum-member')]
        return (name, enumerators)

    def __get_enums(self, dom):
        return filter(None, (self.__get_enum(elem) for elem in dom.getElementsByTagName('enum')))<commit_msg>refactoring<commit_after># -*- coding: utf-8 -*-

import xml.dom.minidom
import model

def _get_struct_name_and_index(struct_list, struct_dict):
    for x in struct_list:
        index = struct_list.index(x)
        struct_dict[x.name] = index

def _sorter(struct_list, index, out_list, struct_dict, lista):
    element = struct_list[index]
    element_name = element.name
    for member_elem in element.members:
        member_type = member_elem.type
        if member_type.startswith('S') and member_type in struct_dict:
            index = struct_dict[member_type]
            x = struct_list[index]
            if x not in out_list:
                out_list.extend(_sorter(struct_list, index, out_list, struct_dict, lista))
        else:
            for x in struct_list:
                if member_type in x.name:
                    if x not in out_list:
                        out_list.append(x)
    if element not in out_list:
        out_list.append(element)
    return lista

def sort_struct(struct_list):
    index = 0
    out_list = []
    struct_dict = {}
    lista = []
    _get_struct_name_and_index(struct_list, struct_dict)
    for struct_elem_index in xrange(len(struct_list)):
        out_list.extend(_sorter(struct_list, struct_elem_index, out_list, struct_dict, lista))
    return out_list

class IsarParser(object):

    primitive_types = {"8 bit integer unsigned": "u8",
                       "16 bit integer unsigned": "u16",
                       "32 bit integer unsigned": "u32",
                       "64 bit integer unsigned": "u64",
                       "8 bit integer signed": "i8",
                       "16 bit integer signed": "i16",
                       "32 bit integer signed": "i32",
                       "64 bit integer signed": "i64",
                       "32 bit float": "r32",
                       "64 bit float": "r64"}

    def __get_struct_members(self, elem):
        members = []
        kname = elem.attributes["name"].value
        ktype = elem.attributes["type"].value
        if elem.getElementsByTagName('dimension'):
            dimension = dict(elem.getElementsByTagName('dimension')[0].attributes.items())
            if "isVariableSize" in dimension:
                type = dimension.get("variableSizeFieldType", "u32")
                name = dimension.get("variableSizeFieldName", kname + "_len")
                members.append(model.Struct.Member(name, type, None, None, None))
                members.append(model.Struct.Member(kname, ktype, True, name, None))
            elif "size" in dimension:
                size = dimension["size"]
                members.append(model.Struct.Member(kname, ktype, True, None, size))
        else:
            members.append(model.Struct.Member(kname, ktype, None, None, None))
        return members

    def __get_struct(self, elem):
        msg = model.Struct()
        msg.name = elem.attributes["name"].value
        msg.members = reduce(lambda x, y: x + y, (self.__get_struct_members(member)
                                                  for member
                                                  in elem.getElementsByTagName('member')), [])
        return msg

    def __get_structs(self, dom):
        return [self.__get_struct(elem)
                for elem
                in dom.getElementsByTagName("struct") + dom.getElementsByTagName("message")
                if elem.hasChildNodes()]

    def __get_enum_member(self, elem):
        value = elem.getAttribute('value')
        value = value if value != "-1" else "0xFFFFFFFF"
        return (elem.attributes["name"].value, value)

    def __get_enum(self, elem):
        name = elem.attributes["name"].value
        enumerators = [self.__get_enum_member(member) for member in elem.getElementsByTagName('enum-member')]
        return (name, enumerators)

    def __get_enums(self, dom):
        return [self.__get_enum(elem) for elem in dom.getElementsByTagName('enum') if elem.hasChildNodes()]
{'fraction_non_alphanumeric': 0.10498687664041995, 'fraction_numerical': 0.010748656417947757, 'mean_word_length': 2.0812475933769736, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '315196', 'n_tokens_mistral': 2301, 'n_tokens_neox': 2219, 'n_words': 547}