<commit_before># -*- coding: utf-8 -*-
# @Author: ZwEin
# @Date:   2016-08-08 11:46:11
# @Last Modified by:   ZwEin
# @Last Modified time: 2016-08-17 14:58:12


from vendor.crf_tokenizer import CrfTokenizer
from common import cleaner
from seed import seeds

DC_NODE_EXT_FEATURE_NAME_CONTENT = 'extracted_text'    # extracted_text
DC_NODE_EXT_FEATURE_NAME_POSTTIME = 'posttime'
DC_NODE_EXT_FEATURE_NAME_CITY = 'city'
DC_NODE_EXT_FEATURE_NAME_TEXT = 'text'
DC_NODE_EXT_FEATURE_NAME_REGION = 'region'
DC_NODE_EXT_FEATURE_NAME_TITLE = 'title'
DC_NODE_EXT_FEATURE_NAME_USERLOCATION = 'userlocation'
DC_NODE_EXT_FEATURE_NAME_PHONENUMBER = 'phonenumber'
DC_NODE_EXT_FEATURE_NAME_SID = 'sid'
DC_NODE_EXT_FEATURE_NAME_OTHERADS = 'otherads'
DC_NODE_EXT_FEATURE_NAME_AGE = 'age'
DC_NODE_EXT_FEATURE_NAME_DOCID = 'doc_id'

DC_NODE_EXT_FEATURE_NAME_EMAIL = 'email'
DC_NODE_EXT_FEATURE_NAME_DRUG_USE = 'drug_use'
DC_NODE_EXT_FEATURE_NAME_PRICE = 'price'
DC_NODE_EXT_FEATURE_NAME_LOCATION = 'location'
DC_NODE_EXT_FEATURE_NAME_PRICE_PER_HOUR = 'price_per_hour'
DC_NODE_EXT_FEATURE_NAME_BUSINESS_TYPE = 'business_type'
DC_NODE_EXT_FEATURE_NAME_URL = 'url'
DC_NODE_EXT_FEATURE_NAME_SERVICES = 'services'
DC_NODE_EXT_FEATURE_NAME_GENDER = 'gender'
DC_NODE_EXT_FEATURE_NAME_PHONE = 'phone'

DC_NODE_EXT_FEATURES = [
    DC_NODE_EXT_FEATURE_NAME_EMAIL,
    DC_NODE_EXT_FEATURE_NAME_DRUG_USE,
    DC_NODE_EXT_FEATURE_NAME_PRICE,
    DC_NODE_EXT_FEATURE_NAME_LOCATION,
    DC_NODE_EXT_FEATURE_NAME_PRICE_PER_HOUR,
    DC_NODE_EXT_FEATURE_NAME_BUSINESS_TYPE,
    DC_NODE_EXT_FEATURE_NAME_URL,
    DC_NODE_EXT_FEATURE_NAME_SERVICES,
    DC_NODE_EXT_FEATURE_NAME_GENDER,
    DC_NODE_EXT_FEATURE_NAME_PHONE,
    DC_NODE_EXT_FEATURE_NAME_AGE
]

DC_NODE_FEATURES = DC_NODE_EXT_FEATURES + sorted(seeds)

class Node(object):

    def __init__(self, content, sid=None, label=None, **attrs):
        self._content = content.encode('utf-8') if content else None
        self._sid = sid
        self._label = label
        self._seeds = seeds
        self._attrs = attrs
        self._features = self.load_features(content)
        self._vector = self.generate_vector()


        print self._features        
        # for attr in sorted(self.attrs.keys()):
        #     print attr, ':', attrs[attr]

    #################################################
    # Clean Content 
    #################################################

    def clean(self, text):
        if not text:
            return ''
        # text = text.encode('ascii', 'ignore')
        text = cleaner.clean_text(text)
        text = text.encode('utf-8')
        # text = unicode(text, 'utf-8')
        # try:
        #     text = text.encode('ascii', 'ignore')
        # except:
        #     text = text.decode('utf-8', 'ignore')
        t = CrfTokenizer()
        t.setRecognizeHtmlEntities(True)
        t.setRecognizeHtmlTags(True)
        t.setSkipHtmlTags(True)
        tokens = t.tokenize(text)

        tokens = [cleaner.clean_token(token) for token in tokens]
        tokens = [_ for _ in set(tokens) if _]

        try:
            str(' '.join(tokens))
        except:
            print text.encode('utf-8')
            print tokens

        return str(' '.join(tokens)).decode('utf-8', 'ignore').encode('ascii', 'ignore')

    #################################################
    # Load Feature
    #################################################
    
    def load_seed_features(self, content):
        if not content:
            return {}
        ans = {}
        content = self.clean(content)
        seed_words = seeds.keys()
        seeds_size = len(seed_words)
        # seed_words.sort()
        tokens = content.split(' ')
        for i in range(seeds_size):
            if seed_words[i] in tokens:
                ans.setdefault(seed_words[i], str(1.0 * float(seeds[seed_words[i]])))
        return ans

    def load_ext_features(self, content):
        def raw2json(raw):
            import ast
            if not raw:
                return []
            try:
                return ast.literal_eval(raw)
            except:
                return raw

        def __ac_func_location(raw):
            if raw:
                return 1.
            return 0.
        def __ac_func_email(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_drug_use(raw):
            if raw:
                return 1.
            return 0.
        def __ac_func_price(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_price_per_hour(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_business_type(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_url(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_services(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_gender(raw):
            if raw:
                for content in raw:
                    if content == 'female':
                        return 1.
            return 0.
        def __ac_func_phone(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_age(raw):
            if raw:
                return len(raw)
            return 0.

        attr_content_funcs = {
            'location': __ac_func_location,
            'email': __ac_func_email,
            'drug_use': __ac_func_drug_use, 
            'price': __ac_func_price, 
            'price_per_hour': __ac_func_price_per_hour, 
            'business_type': __ac_func_business_type, 
            'url': __ac_func_url, 
            'services': __ac_func_services, 
            'gender': __ac_func_gender, <commit_msg>fix bug for url<commit_after># -*- coding: utf-8 -*-
# @Author: ZwEin
# @Date:   2016-08-08 11:46:11
# @Last Modified by:   ZwEin
# @Last Modified time: 2016-08-17 15:06:35


from vendor.crf_tokenizer import CrfTokenizer
from common import cleaner
from seed import seeds

DC_NODE_EXT_FEATURE_NAME_CONTENT = 'extracted_text'    # extracted_text
DC_NODE_EXT_FEATURE_NAME_POSTTIME = 'posttime'
DC_NODE_EXT_FEATURE_NAME_CITY = 'city'
DC_NODE_EXT_FEATURE_NAME_TEXT = 'text'
DC_NODE_EXT_FEATURE_NAME_REGION = 'region'
DC_NODE_EXT_FEATURE_NAME_TITLE = 'title'
DC_NODE_EXT_FEATURE_NAME_USERLOCATION = 'userlocation'
DC_NODE_EXT_FEATURE_NAME_PHONENUMBER = 'phonenumber'
DC_NODE_EXT_FEATURE_NAME_SID = 'sid'
DC_NODE_EXT_FEATURE_NAME_OTHERADS = 'otherads'
DC_NODE_EXT_FEATURE_NAME_AGE = 'age'
DC_NODE_EXT_FEATURE_NAME_DOCID = 'doc_id'

DC_NODE_EXT_FEATURE_NAME_EMAIL = 'email'
DC_NODE_EXT_FEATURE_NAME_DRUG_USE = 'drug_use'
DC_NODE_EXT_FEATURE_NAME_PRICE = 'price'
DC_NODE_EXT_FEATURE_NAME_LOCATION = 'location'
DC_NODE_EXT_FEATURE_NAME_PRICE_PER_HOUR = 'price_per_hour'
DC_NODE_EXT_FEATURE_NAME_BUSINESS_TYPE = 'business_type'
DC_NODE_EXT_FEATURE_NAME_URL = 'url'
DC_NODE_EXT_FEATURE_NAME_SERVICES = 'services'
DC_NODE_EXT_FEATURE_NAME_GENDER = 'gender'
DC_NODE_EXT_FEATURE_NAME_PHONE = 'phone'

DC_NODE_EXT_FEATURES = [
    DC_NODE_EXT_FEATURE_NAME_EMAIL,
    DC_NODE_EXT_FEATURE_NAME_DRUG_USE,
    DC_NODE_EXT_FEATURE_NAME_PRICE,
    DC_NODE_EXT_FEATURE_NAME_LOCATION,
    DC_NODE_EXT_FEATURE_NAME_PRICE_PER_HOUR,
    DC_NODE_EXT_FEATURE_NAME_BUSINESS_TYPE,
    DC_NODE_EXT_FEATURE_NAME_URL,
    DC_NODE_EXT_FEATURE_NAME_SERVICES,
    DC_NODE_EXT_FEATURE_NAME_GENDER,
    DC_NODE_EXT_FEATURE_NAME_PHONE,
    DC_NODE_EXT_FEATURE_NAME_AGE
]

DC_NODE_FEATURES = DC_NODE_EXT_FEATURES + sorted(seeds)

class Node(object):

    def __init__(self, content, sid=None, label=None, **attrs):
        self._content = content.encode('utf-8') if content else None
        self._sid = sid
        self._label = label
        self._seeds = seeds
        self._attrs = attrs
        self._features = self.load_features(content)
        self._vector = self.generate_vector()


        print self._features        
        # for attr in sorted(self.attrs.keys()):
        #     print attr, ':', attrs[attr]

    #################################################
    # Clean Content 
    #################################################

    def clean(self, text):
        if not text:
            return ''
        # text = text.encode('ascii', 'ignore')
        text = cleaner.clean_text(text)
        text = text.encode('utf-8')
        # text = unicode(text, 'utf-8')
        # try:
        #     text = text.encode('ascii', 'ignore')
        # except:
        #     text = text.decode('utf-8', 'ignore')
        t = CrfTokenizer()
        t.setRecognizeHtmlEntities(True)
        t.setRecognizeHtmlTags(True)
        t.setSkipHtmlTags(True)
        tokens = t.tokenize(text)

        tokens = [cleaner.clean_token(token) for token in tokens]
        tokens = [_ for _ in set(tokens) if _]

        try:
            str(' '.join(tokens))
        except:
            print text.encode('utf-8')
            print tokens

        return str(' '.join(tokens)).decode('utf-8', 'ignore').encode('ascii', 'ignore')

    #################################################
    # Load Feature
    #################################################
    
    def load_seed_features(self, content):
        if not content:
            return {}
        ans = {}
        content = self.clean(content)
        seed_words = seeds.keys()
        seeds_size = len(seed_words)
        # seed_words.sort()
        tokens = content.split(' ')
        for i in range(seeds_size):
            if seed_words[i] in tokens:
                ans.setdefault(seed_words[i], str(1.0 * float(seeds[seed_words[i]])))
        return ans

    def load_ext_features(self, content):
        def raw2json(raw):
            import ast
            if not raw:
                return []
            try:
                return ast.literal_eval(raw)
            except:
                return raw

        def __ac_func_location(raw):
            if raw:
                return 1.
            return 0.
        def __ac_func_email(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_drug_use(raw):
            if raw:
                return 1.
            return 0.
        def __ac_func_price(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_price_per_hour(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_business_type(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_url(raw):
            if raw:
                return 1.
            return 0.
        def __ac_func_services(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_gender(raw):
            if raw:
                for content in raw:
                    if content == 'female':
                        return 1.
            return 0.
        def __ac_func_phone(raw):
            if raw:
                return len(raw)
            return 0.
        def __ac_func_age(raw):
            if raw:
                return len(raw)
            return 0.

        attr_content_funcs = {
            'location': __ac_func_location,
            'email': __ac_func_email,
            'drug_use': __ac_func_drug_use, 
            'price': __ac_func_price, 
            'price_per_hour': __ac_func_price_per_hour, 
            'business_type': __ac_func_business_type, 
            'url': __ac_func_url, 
            'services': __ac_func_services, 
            'gender': __ac_func_gender, 
{'fraction_non_alphanumeric': 0.11722897756465148, 'fraction_numerical': 0.008991265627675972, 'mean_word_length': 2.2041152263374486, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5672588', 'n_tokens_mistral': 3826, 'n_tokens_neox': 3698, 'n_words': 693}