<commit_before>#!/usr/bin/env python

# default value for options
default_width=80
default_indent=2

def trim(string):
    """Trim leading and trailing whitespace off of the supplied string"""
    # default value for trimming
    begin=0
    end=string.__len__()-1 # index of last char is one less
                           # than string size
    # determine amount of leading whitespace
    while string[begin].isspace() and begin<end:
        begin=begin+1
    # determine amount of trailing whitespace
    while string[end].isspace() and end>begin:
        end=end-1
    # return shortened result
    if begin+1>=end:
        return ""
    else:
        return string[begin:end+1]

def remove_extra_spaces(string):
    """Condense all whitespace into ' '"""
    # first remove leading and trailing space
    temp_string=trim(string)

    # set up for loop
    is_space=False
    result=""

    # go through entire string, condensing all spaces into " "
    for letter in temp_string:
        if letter.isspace(): # look for a space
            if not is_space: # turn multiple spaces into one
                is_space=True
                result=result+" "
        else: # just copy the result
            is_space=False
            result=result+letter

    # remove the shortened result
    return result

def get_def_ext():
    return "xml"

def get_command_line_doc():
    result = " --width     Specify the width the the document for wrapping. This does not \n"+ \
             "             apply to open or close tags. The default is %d.\n" %default_width + \
             " --indent    Specify the number of spaces for each indentation level. The \n" + \
             "             default is %d." % default_indent
    return result

class format_xml(object):
    """Formatting object that goes through a DOM tree and puts it out
    as a space formatted XML document for easier reading in a simple
    text editor."""
    def __init__(self,main_name,options={},indent=2):
        # variable for tracking indent level
        self.__cur_indent=0

        # option for how wide text can go before wrapping
        try:
            self.__width=int(options.pop("--width"))
        except KeyError:
            self.__width=default_width

        # option for how many spaces to use for each indentation level
        try:
            self.__indent=int(options.pop("--indent"))
        except KeyError:
            self.__indent=default_indent

        # name of definition being formatted
        self.main_name=main_name

    def __setattr__(self,name,value):
        """This does not allow the user to set the value of the
        attributes. This is an immutable object"""
        if name=="width":
            raise AttributeError("Cannot set value of immutable \"width\"")
        if name=="indent":
            raise AttributeError("Cannot set value of immutable \"indent\"")
        if name=="cur_indent":
            raise AttributeError("Cannot set value of immutable \"cur_indent\"")
        self.__dict__[name]=value

    # allow access to the attribute values
    def __getattr__(self,name):
        if name=="width":
            return int(self.__width)
        if name=="indent":
            return int(self.__indent)
        if name=="cur_indent":
            return int(self.__cur_indent)
        return self.__dict__[name]

    def format_head(self):
        """Put a header on the resulting document"""
        return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"

    def format_foot(self):
        """Put a footer on the resulting document"""
        return ""

    def format_data(self,data):
        """Format the data for the output file"""
        # determine the indentation
        indent=self.__cur_indent*" "

        # create the idiot version
        result="%s%s\n" % (indent,data)

        # return the idiot version if it works
        if(result.__len__()<=self.__width):
            return result

        # generate an array of strings of the correct with for formatting
        str_array=[]
        while result.__len__()>self.__width:
            line=result[0:self.__width]
            while not line[-1].isspace():
                line=line[0:-1]
            str_array.append(line+"\n")
            result=indent+trim(result[line.__len__():])
        str_array.append(result+"\n")

        # convert the string array into a single string
        result=""
        for str in str_array:
            result=result+str

        return result

    def format_attributes(self,node):
        """Format the node attributes for the output file"""
        # return empty string if nothing found
        if not node.hasAttributes():
            return ""

        # put together a string of the formatted attributes
        string=""
        for key in node.attributes.keys():
            value=node.attributes[key].value
            string=string+(" %s=\"%s\"" % (key,value))

        return string

    def format_node(self,file,tag,node):
        """Format a node for the output file"""
        # format the attributes
        attr_string=self.format_attributes(node)
        # create the open tag
        file.write("%s<%s%s>\n" % ((" "*self.__cur_indent),tag,attr_string))
        # increase the indent level
        self.__cur_indent=self.__cur_indent+self.__indent
        # recurse down into the tree
        for child in node.childNodes:
            try:
                child_tag=child.tagName<commit_msg>Added wrapping of long opening tags.<commit_after>#!/usr/bin/env python

# default value for options
default_width=80
default_indent=2

def trim(string):
    """Trim leading and trailing whitespace off of the supplied string"""

    # exit early if supplied an empty string
    if not string: return ""

    # default value for trimming
    begin=0
    end=string.__len__()-1 # index of last char is one less
                           # than string size

    # determine amount of leading whitespace
    while string[begin].isspace() and begin<end:
        begin=begin+1
    # determine amount of trailing whitespace
    while string[end].isspace() and end>begin:
        end=end-1
    # return shortened result
    if begin+1>=end:
        return ""
    else:
        return string[begin:end+1]

def remove_extra_spaces(string):
    """Condense all whitespace into ' '"""
    # first remove leading and trailing space
    temp_string=trim(string)

    # set up for loop
    is_space=False
    result=""

    # go through entire string, condensing all spaces into " "
    for letter in temp_string:
        if letter.isspace(): # look for a space
            if not is_space: # turn multiple spaces into one
                is_space=True
                result=result+" "
        else: # just copy the result
            is_space=False
            result=result+letter

    # remove the shortened result
    return result

def wrap_text(string,width,continue_additional_indent=""):
    """This will perform \"word wrapping\" of text based on
    spaces. The default indentation level is determined from the
    number of spaces at the start of the string"""

    # confirm that there is something to wrap
    if string.__len__()<=width:
        return string
    
    # determine the current indentation level
    indent=""
    for char in string:
        if char.isspace():
            indent=indent+char
        else:
            break

    # add the continuation indentation
    indent=indent+continue_additional_indent

    # generate an array of strings of the correct with for formatting
    result=string
    str_array=[]
    while result.__len__()>width:
        line=result[0:width]
        while not line[-1].isspace():
            line=line[0:-1]
        str_array.append(line+"\n")
        result=indent+trim(result[line.__len__():])
    str_array.append(result+"\n")
            
    # convert the string array into a single string
    result=""
    for str in str_array:
        result=result+str

    # return the wrapped string
    return result

def get_def_ext():
    return "xml"

def get_command_line_doc():
    result = " --width     Specify the width the the document for wrapping. This does not \n"+ \
             "             apply to open or close tags. The default is %d.\n" %default_width + \
             " --indent    Specify the number of spaces for each indentation level. The \n" + \
             "             default is %d." % default_indent
    return result

class format_xml(object):
    """Formatting object that goes through a DOM tree and puts it out
    as a space formatted XML document for easier reading in a simple
    text editor."""
    def __init__(self,main_name,options={},indent=2):
        # variable for tracking indent level
        self.__cur_indent=0

        # option for how wide text can go before wrapping
        try:
            self.__width=int(options.pop("--width"))
        except KeyError:
            self.__width=default_width

        # option for how many spaces to use for each indentation level
        try:
            self.__indent=int(options.pop("--indent"))
        except KeyError:
            self.__indent=default_indent

        # name of definition being formatted
        self.main_name=main_name

    def __setattr__(self,name,value):
        """This does not allow the user to set the value of the
        attributes. This is an immutable object"""
        if name=="width":
            raise AttributeError("Cannot set value of immutable \"width\"")
        if name=="indent":
            raise AttributeError("Cannot set value of immutable \"indent\"")
        if name=="cur_indent":
            raise AttributeError("Cannot set value of immutable \"cur_indent\"")
        self.__dict__[name]=value

    # allow access to the attribute values
    def __getattr__(self,name):
        if name=="width":
            return int(self.__width)
        if name=="indent":
            return int(self.__indent)
        if name=="cur_indent":
            return int(self.__cur_indent)
        return self.__dict__[name]

    def format_head(self):
        """Put a header on the resulting document"""
        return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"

    def format_foot(self):
        """Put a footer on the resulting document"""
        return ""

    def format_data(self,data):
        """Format the data for the output file"""
        # determine the indentation
        indent=self.__cur_indent*" "

        # create the idiot version
        result="%s%s\n" % (indent,data)

        return wrap_text(result,self.__width)
#
#        # return the idiot version if it works
#        if(result.__len__()<=self.__width):
#            return result
#
#        # generate an array of strings of the correct with for formatting
#        str_array=[]
#        while result.__len__()>self.__width:
#            line=result[0:self.__width]
#            while not line[-1].isspace():
#                line=line[0:-1]
#            str_array.append(line+"\n")
#            result=indent+trim(result[line.__len__():])
#        str_array.append(result+"\n")
#
#        # convert the string array into a single string
#        result=""
#        for str in str_array:
#            result=result+str
#
#        return result

    def format_attributes(self,node):
        """Format the node attributes for the output file"""
        # return empty string if nothing found
        if not node.hasAttributes():
            return ""

        # put together a string of the formatted attributes
        string=""
        for key in node.attributes.keys():
            value=node.attributes[key].value
            string=string+(" %s=\"%s\"" % (key,value))

        return string

    def format_node(self,file,tag,node):
        """Format a node for the output file"""
        # format the attributes
        attr_string=self.format_attributes(node)
        # create the open tag
        file.write(wrap_text("%s<%s%s>\n" % ((" "*self.__cur_indent),tag,attr_string),self.__width,(" "*(self.__indent+2))))
        # increase the indent level
        self.__cur_indent=self.__cur_indent+self.__indent
        # recurse down into the tree
        for child in node.childNodes:
            try:
                child_tag=child.tagName
{'fraction_non_alphanumeric': 0.09677156367193869, 'fraction_numerical': 0.003342572965922061, 'mean_word_length': 2.2264597580220937, 'pattern_counts': {'":': 12, '<': 12, '<?xml version=': 2, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '817579', 'n_tokens_mistral': 3304, 'n_tokens_neox': 3217, 'n_words': 1081}