<commit_before>from lxml import etree
from StringIO import StringIO
import numpy as np
import json


class tree(object):

    def __init__(self):
        pass

    def get_depth(self, root):
        if len(root) == 0:
            return 0
        return max(self.get_depth(child) + 1 for child in root.getchildren())

    def get_size(self, root):
        if len(root) == 0:
            return 0
        return sum(1 + self.get_size(child) for child in root.getchildren())

    def tree_match(self, t1, t2, matched):
        # Computes the Simple Tree Matching distance between trees and
        # stores the matching(in pairs of form (element1, element2) in "matched" list passed as argument
        if t1 is None or t2 is None:
            return 0
        
        if t1.tag != t2.tag:
            return 0

        flag = 0
        # Recording alignment as nodes are matched
        # Not an elegant way, bad time complexity. Each call of tree_match calls this. Think
        # of some workaround for this.
        for matching in matched:
            if t1 in matching or t2 in matching:
                flag = 1
        if flag == 0:
            # To remove unncecesary script tags matching
            if t1.tag != "script" and t2.tag != "script":
                matched.append([t1, t2])
        child1 = t1.getchildren()
        child2 = t2.getchildren()
        m = len(t1)
        n = len(t2)
        M = np.zeros((m+1, n+1), np.int)
        for i in range(1, m+1):
            for j in range(1, n+1):
                M[i][j] = max(M[i][j-1], M[i-1][j], M[i-1][j-1] +
                              self.tree_match(child1[i-1], child2[j-1], matched))
        return (M[m][n]+1)

    def tree_alignment(self):
        pass

if __name__ == "__main__":
    # Add these to unittestcase in test folder
    parser = etree.HTMLParser()
    s1 = """<root>
            <a></a>
            <b>
                <c></c>
                <d></d>
            </b>
            </root>
    """
    s5 = """<c> <b> <a> </a> </b> </c>"""
    s6 = """<c><a></a> <b></b></c> """
    s2 = """<root>
            <e></e>
            <b>
                <c></c>
                <d></d>
            </b>
            </root>
    """
    s3 = """<root>
                <b></b>
                <c></c>
            </root>
        """
    s4 = """<root>
                <c></c>
            </root>
        """

    tree1 = etree.XML(s5)
    tree2 = etree.XML(s6)
    tree = tree()
    print "===========  Testing  ========="
    ans = []
    print tree.tree_match(tree1, tree2, ans)
    print ans
    
    tree1 = etree.parse("../tests/test.html")
    print "==============================================="<commit_msg>test removed from tree.py and added to tests/test_tree.py<commit_after>from lxml import etree
from StringIO import StringIO
import numpy as np
import json


class tree(object):

    def __init__(self):
        pass

    def get_depth(self, root):
        if len(root) == 0:
            return 0
        return max(self.get_depth(child) + 1 for child in root.getchildren())

    def get_size(self, root):
        if len(root) == 0:
            return 0
        return sum(1 + self.get_size(child) for child in root.getchildren())

    def tree_match(self, t1, t2, matched):
        # Computes the Simple Tree Matching distance between trees and
        # stores the matching(in pairs of form (element1, element2) in "matched" list passed as argument
        if t1 is None or t2 is None:
            return 0
        
        if t1.tag != t2.tag:
            return 0

        flag = 0
        # Recording alignment as nodes are matched
        # Not an elegant way, bad time complexity. Each call of tree_match calls this. Think
        # of some workaround for this.
        for matching in matched:
            if t1 in matching or t2 in matching:
                flag = 1
        if flag == 0:
            # To remove unncecesary script tags matching
            if t1.tag != "script" and t2.tag != "script":
                matched.append([t1, t2])
        child1 = t1.getchildren()
        child2 = t2.getchildren()
        m = len(t1)
        n = len(t2)
        M = np.zeros((m+1, n+1), np.int)
        for i in range(1, m+1):
            for j in range(1, n+1):
                M[i][j] = max(M[i][j-1], M[i-1][j], M[i-1][j-1] +
                              self.tree_match(child1[i-1], child2[j-1], matched))
        return (M[m][n]+1)

    def tree_alignment(self):
        pass

if __name__ == "__main__":
    # Add these to unittestcase in test folder
    parser = etree.HTMLParser()
   tree = tree()
    
    tree1 = etree.parse("../tests/test.html")
    print "==============================================="
{'fraction_non_alphanumeric': 0.1465791145943656, 'fraction_numerical': 0.022452870154628258, 'mean_word_length': 1.6738391845979614, 'pattern_counts': {'":': 4, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4174170', 'n_tokens_mistral': 1530, 'n_tokens_neox': 1470, 'n_words': 457}