<commit_before>import re
from HTMLParser import HTMLParser

class WebPageNode:
    def __init__(self, tag, attrs):
        """
        Initialise a WebPageNode object
        """

        self.tag_ = tag
        self.attrs_ = attrs
        
        self.nodes_ = list()
        self.data_ = None
    
    def get_tags(self):
        return self.tags_

    def get_attrs(self):
        return dict(self.attrs_)

    def set_data(self, data):
        self.data_ = data

    @staticmethod
    def css_selector_to_query_elts(css_selector_query):
        """
        Convert CSS selector to a list of subqueries
        cf. CSS Selectors: http://www.w3schools.com/cssref/css_selectors.asp

        CSS selectors which not implemented:
        + :......
        + elt1 , elt2
        + elt1 > elt2
        + elt1 ~ elt2
        + elt1 + elt2

        eg.: head meta[description]
        eg.: p.myclass
        eg.: a[target=blank_]
        """
        
        regex_tag = re.compile(r'^([^\.\[\]#]*)')
        regex_id = re.compile(r'#([^\.\[\]#]*)')
        regex_class = re.compile(r'\.([^\.\[\]#]*)')
        regex_attr = re.compile(r'\[([^\.\[\]#=\*~\|\^$]*)([\*~\|\^$]?="([^"]*)"|[\*~\|\^$]?=\'([^\']*)\'|[\*~\|\^$]?=([^\]]+)|)\]') 
        
        query_elts = list()
        query_raw_elts = css_selector_query.split(' ');
        for raw_elt in query_raw_elts:
            # Tag name
            m = regex_tag.search(raw_elt)
            if m:
                tag = m.group(1)
            else:
                tag = None
            
            # Class & id
            attrs = dict()
            m = regex_id.search(raw_elt)
            if m:
                attrs["id"] = {"value": m.group(1), "type": "="}
            m = regex_class.search(raw_elt)
            if m:
                attrs["class"] = {"value": m.group(1), "type": "="}

            # Attribute
            m = regex_attr.findall(raw_elt)
            for attr_in_query in m:
                # attr_in_query = (attr_name, value with equal and quotations marks, value (if ="value"), value (if ='value'), value (if =value))
                if attr_in_query[1]:
                    if attr_in_query[2]:
                        attrs[attr_in_query[0]] = {"value": attr_in_query[2], "type": attr_in_query[1][0]}
                    elif attr_in_query[3]:
                        attrs[attr_in_query[0]] = {"value": attr_in_query[3], "type": attr_in_query[1][0]}
                    else:
                        attrs[attr_in_query[0]] = {"value": attr_in_query[4], "type": attr_in_query[1][0]}
                else:
                    attrs[attr_in_query[0]] = None
            
            print tag
            print attrs
            query_elts.append({"tag": tag, "attrs": attrs})
        return query_elts
    
    def is_fit_query(self, query_elt):
        """
        Return True if self fits query_elt
        """
        
        # Tag?
        try:<commit_msg>BUG: WebPageNode.attrs_ were not stored in dict<commit_after>import re
from HTMLParser import HTMLParser

class WebPageNode:
    def __init__(self, tag, attrs):
        """
        Initialise a WebPageNode object
        """

        self.tag_ = tag
        self.attrs_ = dict(attrs)
        
        self.nodes_ = list()
        self.data_ = None
    
    def get_tags(self):
        return self.tags_

    def get_attrs(self):
        return dict(self.attrs_)

    def set_data(self, data):
        self.data_ = data

    @staticmethod
    def css_selector_to_query_elts(css_selector_query):
        """
        Convert CSS selector to a list of subqueries
        cf. CSS Selectors: http://www.w3schools.com/cssref/css_selectors.asp

        CSS selectors which not implemented:
        + :......
        + elt1 , elt2
        + elt1 > elt2
        + elt1 ~ elt2
        + elt1 + elt2

        eg.: head meta[description]
        eg.: p.myclass
        eg.: a[target=blank_]
        """
        
        regex_tag = re.compile(r'^([^\.\[\]#]*)')
        regex_id = re.compile(r'#([^\.\[\]#]*)')
        regex_class = re.compile(r'\.([^\.\[\]#]*)')
        regex_attr = re.compile(r'\[([^\.\[\]#=\*~\|\^$]*)([\*~\|\^$]?="([^"]*)"|[\*~\|\^$]?=\'([^\']*)\'|[\*~\|\^$]?=([^\]]+)|)\]') 
        
        query_elts = list()
        query_raw_elts = css_selector_query.split(' ');
        for raw_elt in query_raw_elts:
            # Tag name
            m = regex_tag.search(raw_elt)
            if m:
                tag = m.group(1)
            else:
                tag = None
            
            # Class & id
            attrs = dict()
            m = regex_id.search(raw_elt)
            if m:
                attrs["id"] = {"value": m.group(1), "type": "="}
            m = regex_class.search(raw_elt)
            if m:
                attrs["class"] = {"value": m.group(1), "type": "="}

            # Attribute
            m = regex_attr.findall(raw_elt)
            for attr_in_query in m:
                # attr_in_query = (attr_name, value with equal and quotations marks, value (if ="value"), value (if ='value'), value (if =value))
                if attr_in_query[1]:
                    if attr_in_query[2]:
                        attrs[attr_in_query[0]] = {"value": attr_in_query[2], "type": attr_in_query[1][0]}
                    elif attr_in_query[3]:
                        attrs[attr_in_query[0]] = {"value": attr_in_query[3], "type": attr_in_query[1][0]}
                    else:
                        attrs[attr_in_query[0]] = {"value": attr_in_query[4], "type": attr_in_query[1][0]}
                else:
                    attrs[attr_in_query[0]] = None
            
            query_elts.append({"tag": tag, "attrs": attrs})
        return query_elts
    
    def is_fit_query(self, query_elt):
        """
        Return True if self fits query_elt
        """
        
        # Tag?
        try:
{'fraction_non_alphanumeric': 0.15690946221919674, 'fraction_numerical': 0.009530292716133424, 'mean_word_length': 1.544155844155844, 'pattern_counts': {'":': 24, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2057330', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1871, 'n_words': 379}