<commit_before>import sys

stdout = sys.stdout
reload(sys)
sys.setdefaultencoding('utf-8')
sys.stdout = stdout
# import all extractors
from spacy_extractors import age_extractor as spacy_age_extractor
from spacy_extractors import social_media_extractor as spacy_social_media_extractor
from spacy_extractors import date_extractor as spacy_date_extractor
from spacy_extractors import address_extractor as spacy_address_extractor
from spacy_extractors import customized_extractor as custom_spacy_extractor
from data_extractors import landmark_extraction
from data_extractors import dictionary_extractor
from data_extractors import regex_extractor
from data_extractors import height_extractor
from data_extractors import weight_extractor
from data_extractors import address_extractor
from data_extractors import age_extractor
from data_extractors import table_extractor
from data_extractors import url_country_extractor
from data_extractors import geonames_extractor
from data_extractors.digPhoneExtractor import phone_extractor
from data_extractors.digEmailExtractor import email_extractor
from data_extractors.digPriceExtractor import price_extractor
from data_extractors.digReviewIDExtractor import review_id_extractor
from data_extractors import date_parser
from classifiers import country_classifier
from structured_extractors import ReadabilityExtractor, TokenizerExtractor, FaithfulTokenizerExtractor
import json
import gzip
import re
import spacy
import codecs
from jsonpath_rw import parse
import time
import collections
import numbers
from tldextract import tldextract
import pickle
import copy
from collections import OrderedDict
import sys

_KNOWLEDGE_GRAPH = "knowledge_graph"
_EXTRACTION_POLICY = 'extraction_policy'
_KEEP_EXISTING = 'keep_existing'
_REPLACE = 'replace'
_ERROR_HANDLING = 'error_handling'
_IGNORE_EXTRACTION = 'ignore_extraction'
_IGNORE_DOCUMENT = 'ignore_document'
_RAISE_ERROR = 'raise_error'
_CITY_NAME = 'city_name'
_STATE = 'state'
_COUNTRY = 'country'
_CONTENT_EXTRACTION = 'content_extraction'
_SPACY_EXTRACTION = 'spacy_extraction'
_RAW_CONTENT = 'raw_content'
_INPUT_PATH = 'input_path'
_READABILITY = 'readability'
_LANDMARK = 'landmark'
_TITLE = 'title'
_DESCRIPTION = "description"
_STRICT = 'strict'
_FIELD_NAME = 'field_name'
_CONTENT_STRICT = 'content_strict'
_CONTENT_RELAXED = 'content_relaxed'
_YES = 'yes'
_NO = 'no'
_RECALL_PRIORITY = 'recall_priority'
_INFERLINK_EXTRACTIONS = 'inferlink_extractions'
_LANDMARK_THRESHOLD = 'landmark_threshold'
_LANDMARK_RULES = 'landmark_rules'
_URL = 'url'
_AGE = 'age'
_POSTING_DATE = 'posting_date'
_SOCIAL_MEDIA = 'social_media'
_ADDRESS = 'address'
_RESOURCES = 'resources'
_SPACY_FIELD_RULES = "spacy_field_rules"
_DATA_EXTRACTION = 'data_extraction'
_FIELDS = 'fields'
_EXTRACTORS = 'extractors'
_TOKENS = 'tokens'
_TOKENS_ORIGINAL_CASE = "tokens_original_case"
_SIMPLE_TOKENS = 'simple_tokens'
_SIMPLE_TOKENS_ORIGINAL_CASE = 'simple_tokens_original_case'
_TEXT = 'text'
_DICTIONARY = 'dictionary'
_PICKLES = 'pickle'
_NGRAMS = 'ngrams'
_JOINER = 'joiner'
_PRE_FILTER = 'pre_filter'
_POST_FILTER = 'post_filter'
_PRE_PROCESS = "pre_process"
_TABLE = "table"
_STOP_WORDS = "stop_words"
_GEONAMES = "geonames"
_STATE_TO_COUNTRY = "state_to_country"
_STATE_TO_CODES_LOWER = "state_to_codes_lower"
_POPULATED_PLACES = "populated_places"
_POPULATED_CITIES = "populated_cities"
_CASE_SENSITIVE = 'case_sensitive'

_EXTRACT_USING_DICTIONARY = "extract_using_dictionary"
_EXTRACT_USING_REGEX = "extract_using_regex"
_EXTRACT_FROM_LANDMARK = "extract_from_landmark"
_EXTRACT_PHONE = "extract_phone"
_EXTRACT_EMAIL = "extract_email"
_EXTRACT_PRICE = "extract_price"
_EXTRACT_HEIGHT = "extract_height"
_EXTRACT_WEIGHT = "extract_weight"
_EXTRACT_ADDRESS = "extract_address"
_EXTRACT_AGE = "extract_age"

_CONFIG = "config"
_DICTIONARIES = "dictionaries"
_INFERLINK = "inferlink"
_HTML = "html"

_SEGMENT_TITLE = "title"
_SEGMENT_INFERLINK_DESC = "inferlink_description"
_SEGMENT_OTHER = "other_segment"

_METHOD_INFERLINK = "inferlink"

_SOURCE_TYPE = "source_type"
_OBFUSCATION = "obfuscation"

_INCLUDE_CONTEXT = "include_context"
_KG_ENHANCEMENT = "kg_enhancement"
_DOCUMENT_ID = "document_id"
_TLD = 'tld'
_FEATURE_COMPUTATION = "feature_computation"


class Core(object):
    def __init__(self, extraction_config=None, debug=False, load_spacy=False):
        self.extraction_config = extraction_config
        self.debug = debug
        self.html_title_regex = r'<title>(.*?)</title>'
        self.tries = dict()
        self.pickles = dict()
        self.jobjs = dict()
        self.global_extraction_policy = None
        self.global_error_handling = None
        # to make sure we do not parse json_paths more times than needed, we define the following 2 properties
        self.content_extraction_path = None
        self.data_extraction_path = dict()
        if load_spacy:
            self.prep_spacy()
        else:
            self.nlp = None
        self.country_code_dict = None
        self.matchers = dict()
        self.geonames_dict = None
        self.state_to_country_dict = None
        self.state_to_codes_lower_dict = None
        self.populated_cities = None

    """ Define all API methods """

    def process(self, doc, create_knowledge_graph=False):
        try:
            # print 'Now Processing url: {}, doc_id: {}'.format(doc['url'], doc['doc_id'])
            if self.extraction_config:
                doc_id = None
                if _DOCUMENT_ID in self.extraction_config:
                    doc_id_field = self.extraction_config[_DOCUMENT_ID]
                    if doc_id_field in doc:
                        doc_id = doc[doc_id_field]
                        doc[_DOCUMENT_ID] = doc_id
                    else:
                        raise KeyError('{} not found in the input document'.format(doc_id_field))
                if _EXTRACTION_POLICY in self.extraction_config:
                    self.global_extraction_policy = self.extraction_config[_EXTRACTION_POLICY]
                if _ERROR_HANDLING in self.extraction_config:
                    self.global_error_handling = self.extraction_config[_ERROR_HANDLING]

                """Handle content extraction first aka Phase 1"""
                if _CONTENT_EXTRACTION in self.extraction_config:
                    if _CONTENT_EXTRACTION not in doc:
                        doc[_CONTENT_EXTRACTION] = dict()
                    ce_config = self.extraction_config[_CONTENT_EXTRACTION]
                    html_path = ce_config[_INPUT_PATH] if _INPUT_PATH in ce_config else None
                    if not html_path:
                        raise KeyError('{} not found in extraction_config'.format(_INPUT_PATH))

                    if not self.content_extraction_path:
                        start_time = time.time()
                        self.content_extraction_path = parse(html_path)
                        time_taken = time.time() - start_time
                        if self.debug:
                            print 'time taken to process parse %s' % time_taken
                    start_time = time.time()
                    matches = self.content_extraction_path.find(doc)
                    time_taken = time.time() - start_time
                    if self.debug:
                        print 'time taken to process matches %s' % time_taken
                    extractors = ce_config[_EXTRACTORS]
                    for index in range(len(matches)):
                        for extractor in extractors.keys():
                            if extractor == _READABILITY:
                                # TODO REMOVE THIS HACK
                                if len(matches[index].value) < 700000 and 'amigobulls.com' not in doc['url'] and 'kulakowka.com' not in doc['url']:
                                    re_extractors = extractors[extractor]
                                    if isinstance(re_extractors, dict):
                                        re_extractors = [re_extractors]
                                    for re_extractor in re_extractors:
                                        doc[_CONTENT_EXTRACTION] = self.run_readability(doc[_CONTENT_EXTRACTION],
                                                                                        matches[index].value,
                                                                                        re_extractor)
                                else:
                                    print 'Large document not running READABILITY, doc_id: {}'.format(doc['doc_id'])
                            elif extractor == _TITLE:
                                doc[_CONTENT_EXTRACTION] = self.run_title(doc[_CONTENT_EXTRACTION],
                                                                          matches[index].value,
                                                                          extractors[extractor])
                            elif extractor == _LANDMARK:
                                doc[_CONTENT_EXTRACTION] = self.run_landmark(doc[_CONTENT_EXTRACTION],
                                                                             matches[index].value,
                                                                             extractors[extractor], doc[_URL])
                            elif extractor == _TABLE:
                                doc[_CONTENT_EXTRACTION] = self.run_table_extractor(doc[_CONTENT_EXTRACTION],
                                                                                    matches[index].value,
                                                                                    extractors[extractor])
                    # Add the url as segment as well
                    if _URL in doc and doc[_URL] and doc[_URL].strip() != '':
                        doc[_CONTENT_EXTRACTION][_URL] = dict()
                        doc[_CONTENT_EXTRACTION][_URL][_TEXT] = doc[_URL]
                        doc[_TLD] = self.extract_tld(doc[_URL])

                """Phase 2: The Data Extraction"""
                if _DATA_EXTRACTION in self.extraction_config:
                    de_configs = self.extraction_config[_DATA_EXTRACTION]
                    if isinstance(de_configs, dict):
                        de_configs = [de_configs]

                    for i in range(len(de_configs)):
                        de_config = de_configs[i]
                        input_paths = de_config[_INPUT_PATH] if _INPUT_PATH in de_config else None
                        if not input_paths:
                            raise KeyError('{} not found for data extraction in extraction_config'.format(_INPUT_PATH))

                        if not isinstance(input_paths, list):
                            input_paths = [input_paths]

                        for input_path in input_paths:
                            if _FIELDS in de_config:
                                if input_path not in self.data_extraction_path:
                                    self.data_extraction_path[input_path] = parse(input_path)
                                matches = self.data_extraction_path[input_path].find(doc)
                                for match in matches:
                                    # First rule of DATA Extraction club: Get tokens
                                    # Get the crf tokens
                                    if _TEXT in match.value:
                                        if _TOKENS_ORIGINAL_CASE not in match.value:
                                            match.value[_TOKENS_ORIGINAL_CASE] = self.extract_crftokens(
                                                match.value[_TEXT],
                                                lowercase=False)
                                        if _TOKENS not in match.value:
                                            match.value[_TOKENS] = self.crftokens_to_lower(
                                                match.value[_TOKENS_ORIGINAL_CASE])
                                        if _SIMPLE_TOKENS not in match.value:
                                            match.value[_SIMPLE_TOKENS] = self.extract_tokens_from_crf(
                                                match.value[_TOKENS])
                                        if _SIMPLE_TOKENS_ORIGINAL_CASE not in match.value:
                                            match.value[_SIMPLE_TOKENS_ORIGINAL_CASE] = self.extract_tokens_from_crf(
                                                match.value[_TOKENS_ORIGINAL_CASE])
                                            # if _TOKENS not in match.value:
                                            #     match.value[_TOKENS] = self.extract_crftokens(match.value[_TEXT])
                                            # if _SIMPLE_TOKENS not in match.value:
                                            #     match.value[_SIMPLE_TOKENS] = self.extract_tokens_from_crf(match.value[_TOKENS])
                                    fields = de_config[_FIELDS]
                                    for field in fields.keys():
                                        if field != '*':
                                            """
                                                Special case for inferlink extractions:
                                                For eg, We do not want to extract name from inferlink_posting-date #DUH
                                            """
                                            run_extractor = True
                                            full_path = str(match.full_path)
                                            segment = self.determine_segment(full_path)
                                            if _INFERLINK in full_path:
                                                if field not in full_path:
                                                    run_extractor = False
                                                if _DESCRIPTION in full_path or _TITLE in full_path:
                                                    run_extractor = True
                                            if run_extractor:
                                                if _EXTRACTORS in fields[field]:
                                                    extractors = fields[field][_EXTRACTORS]
                                                    for extractor in extractors.keys():
                                                        try:
                                                            foo = getattr(self, extractor)
                                                        except:
                                                            foo = None
                                                        if foo:
                                                            # score is 1.0 because every method thinks it is the best
                                                            score = 1.0
                                                            method = extractor
                                                            if _CONFIG not in extractors[extractor]:
                                                                extractors[extractor][_CONFIG] = dict()
                                                            extractors[extractor][_CONFIG][_FIELD_NAME] = field
                                                            ep = self.determine_extraction_policy(extractors[extractor])
                                                            if extractor == _EXTRACT_FROM_LANDMARK:
                                                                if _INFERLINK_EXTRACTIONS in full_path and field in full_path:
                                                                    method = _METHOD_INFERLINK
                                                                    if self.check_if_run_extraction(match.value, field,
                                                                                                    extractor,
                                                                                                    ep):

                                                                        results = foo(doc,
                                                                                      extractors[extractor][_CONFIG])
                                                                        if results:
                                                                            self.add_data_extraction_results(
                                                                                match.value,
                                                                                field,
                                                                                extractor,
                                                                                self.add_origin_info(
                                                                                    results,
                                                                                    method,
                                                                                    segment,
                                                                                    score,
                                                                                    doc_id))
                                                                            if create_knowledge_graph:
                                                                                self.create_knowledge_graph(doc, field,
                                                                                                            results)
                                                            else:
                                                                if self.check_if_run_extraction(match.value, field,
                                                                                                extractor,
                                                                                                ep):
                                                                    results = foo(match.value,
                                                                                  extractors[extractor][_CONFIG])
                                                                    if results:
                                                                        self.add_data_extraction_results(match.value,
                                                                                                         field,
                                                                                                         extractor,
                                                                                                         self.add_origin_info(
                                                                                                             results,
                                                                                                             method,
                                                                                                             segment,
                                                                                                             score,
                                                                                                             doc_id))
                                                                        if create_knowledge_graph:
                                                                            self.create_knowledge_graph(doc, field,
                                                                                                        results)
                                        else:  # extract whatever you can!
                                            if _EXTRACTORS in fields[field]:
                                                extractors = fields[field][_EXTRACTORS]
                                                for extractor in extractors.keys():
                                                    try:
                                                        foo = getattr(self, extractor)
                                                    except Exception as e:
                                                        foo = None
                                                    if foo:
                                                        # score is 1.0 because every method thinks it is the best
                                                        score = 1.0
                                                        method = extractor
                                                        if _CONFIG not in extractors[extractor]:
                                                            extractors[extractor][_CONFIG] = dict()
                                                        ep = self.determine_extraction_policy(extractors[extractor])
                                                        if extractor == _EXTRACT_FROM_LANDMARK:
                                                            if _INFERLINK_EXTRACTIONS in full_path and field in full_path:
                                                                method = _METHOD_INFERLINK
                                                                if self.check_if_run_extraction(match.value, field,
                                                                                                extractor,
                                                                                                ep):

                                                                    results = foo(doc, extractors[extractor][_CONFIG])
                                                                    if results:
                                                                        self.add_data_extraction_results(match.value,
                                                                                                         field,
                                                                                                         extractor,
                                                                                                         self.add_origin_info(
                                                                                                             results,
                                                                                                             method,
                                                                                                             segment,
                                                                                                             score,
                                                                                                             doc_id))
                                                                        if create_knowledge_graph:
                                                                            self.create_knowledge_graph(doc, field,
                                                                                                        results)
                                                        else:
                                                            results = foo(match.value,
                                                                          extractors[extractor][_CONFIG])
                                                            if results:
                                                                for f, res in results.items():
                                                                    # extractors[extractor][_CONFIG][_FIELD_NAME] = f
                                                                    self.add_data_extraction_results(match.value, f,
                                                                                                     extractor,
                                                                                                     self.add_origin_info(
                                                                                                         res,
                                                                                                         method,
                                                                                                         segment,
                                                                                                         score, doc_id))
                                                                    if create_knowledge_graph:
                                                                        self.create_knowledge_graph(doc, f, res)
                                                    else:
                                                        print('method {} not found!'.format(extractor))

                """Optional Phase 3: Knowledge Graph Enhancement"""
                if _KG_ENHANCEMENT in self.extraction_config:
                    kg_configs = self.extraction_config[_KG_ENHANCEMENT]
                    if isinstance(kg_configs, dict):
                        kg_configs = [kg_configs]

                    for i in range(len(kg_configs)):
                        kg_config = kg_configs[i]
                        input_paths = kg_config[_INPUT_PATH] if _INPUT_PATH in kg_config else None
                        if not input_paths:
                            raise KeyError(
                                '{} not found for knowledge graph enhancement in extraction_config'.format(_INPUT_PATH))

                        if not isinstance(input_paths, list):
                            input_paths = [input_paths]

                        for input_path in input_paths:
                            if _FIELDS in kg_config:
                                if input_path not in self.data_extraction_path:
                                    self.data_extraction_path[input_path] = parse(input_path)
                                matches = self.data_extraction_path[input_path].find(doc)
                                for match in matches:
                                    fields = kg_config[_FIELDS]
                                    try:
                                        sorted_fields = self.sort_dictionary_by_fields(fields)
                                    except:
                                        raise ValueError('Please ensure there is a priority added to every field in '
                                                         'knowledge_graph  enhancement and the priority is an int')
                                    for i in range(0, len(sorted_fields)):
                                        field = sorted_fields[i][0]
                                        if _EXTRACTORS in fields[field]:
                                            extractors = fields[field][_EXTRACTORS]
                                            for extractor in extractors.keys():
                                                try:
                                                    foo = getattr(self, extractor)
                                                except:
                                                    foo = None
                                                if foo:
                                                    if _CONFIG not in extractors[extractor]:
                                                        extractors[extractor][_CONFIG] = dict()
                                                    extractors[extractor][_CONFIG][_FIELD_NAME] = field
                                                    results = foo(match.value, extractors[extractor][_CONFIG])
                                                    if results:
                                                        # doc[_KNOWLEDGE_GRA][field] = results
                                                        self.create_knowledge_graph(doc, field, results)

                """Optional Phase 4: feature computation"""
                if _FEATURE_COMPUTATION in self.extraction_config:
                    kg_configs = self.extraction_config[_FEATURE_COMPUTATION]
                    if isinstance(kg_configs, dict):
                        kg_configs = [kg_configs]

                    for i in range(len(kg_configs)):
                        kg_config = kg_configs[i]
                        input_paths = kg_config[_INPUT_PATH] if _INPUT_PATH in kg_config else None
                        if not input_paths:
                            raise KeyError(
                                '{} not found for feature computation in extraction_config'.format(_INPUT_PATH))

                        if not isinstance(input_paths, list):
                            input_paths = [input_paths]

                        for input_path in input_paths:
                            if _FIELDS in kg_config:
                                if input_path not in self.data_extraction_path:
                                    self.data_extraction_path[input_path] = parse(input_path)
                                matches = self.data_extraction_path[input_path].find(doc)
                                for match in matches:
                                    fields = kg_config[_FIELDS]
                                    for field in fields.keys():
                                        if _EXTRACTORS in fields[field]:
                                            extractors = fields[field][_EXTRACTORS]
                                            for extractor in extractors.keys():
                                                try:
                                                    foo = getattr(self, extractor)
                                                except:
                                                    foo = None
                                                if foo:
                                                    if _CONFIG not in extractors[extractor]:
                                                        extractors[extractor][_CONFIG] = dict()
                                                    extractors[extractor][_CONFIG][_FIELD_NAME] = field
                                                    results = foo(match.value, extractors[extractor][_CONFIG])
                                                    if results:
                                                        # doc[_KNOWLEDGE_GRAPH][field] = results
                                                        self.create_knowledge_graph(doc, field, results)

                if _KNOWLEDGE_GRAPH in doc and doc[_KNOWLEDGE_GRAPH]:
                    doc[_KNOWLEDGE_GRAPH] = self.reformat_knowledge_graph(doc[_KNOWLEDGE_GRAPH])
                    """ Add title and description as fields in the knowledge graph as well"""
                    doc = Core.rearrange_description(doc)
                    doc = Core.rearrange_title(doc)
        except Exception as e:
            print e
            print 'Failed doc:', doc['doc_id']
            raise e
        # print 'DONE url: {}, doc_id: {}'.format(doc['url'], doc['doc_id'])
        return doc

    @staticmethod
    def rearrange_description(doc):
        method = 'rearrange_description'
        description = None
        segment = ''
        if _CONTENT_EXTRACTION in doc:
            ce = doc[_CONTENT_EXTRACTION]
            if _INFERLINK_EXTRACTIONS in ce:
                if _DESCRIPTION in ce[_INFERLINK_EXTRACTIONS]:
                    description = ce[_INFERLINK_EXTRACTIONS][_DESCRIPTION][_TEXT]
                    segment = _INFERLINK
            if not description or description.strip() == '':
                if _CONTENT_STRICT in ce:
                    description = ce[_CONTENT_STRICT][_TEXT]
                    segment = _CONTENT_STRICT

            if description and description != '':
                if _KNOWLEDGE_GRAPH not in doc:
                    doc[_KNOWLEDGE_GRAPH] = dict()
                doc[_KNOWLEDGE_GRAPH][_DESCRIPTION] = list()
                o = dict()
                o['value'] = description
                o['key'] = 'description'
                o['confidence'] = 1
                o['provenance'] = [Core.custom_provenance_object(method, segment, doc[_DOCUMENT_ID])]
                doc[_KNOWLEDGE_GRAPH][_DESCRIPTION].append(o)
        return doc

    @staticmethod
    def sort_dictionary_by_fields(dictionary):
        sorted_d = OrderedDict(sorted(dictionary.iteritems(), key=lambda x: x[1]['priority']))
        return sorted_d.items()

    @staticmethod
    def custom_provenance_object(method, segment, document_id):
        prov = dict()
        prov['method'] = method
        prov['source'] = dict()
        prov['source']['segment'] = segment
        prov['source'][_DOCUMENT_ID] = document_id
        return prov

    @staticmethod
    def rearrange_title(doc):
        method = 'rearrange_title'
        title = None
        segment = ''
        if _CONTENT_EXTRACTION in doc:
            ce = doc[_CONTENT_EXTRACTION]
            if _INFERLINK_EXTRACTIONS in ce:
                if _TITLE in ce[_INFERLINK_EXTRACTIONS]:
                    title = ce[_INFERLINK_EXTRACTIONS][_TITLE][_TEXT]
                    segment = _INFERLINK
            if not title or title.strip() == '':
                if _TITLE in ce:
                    title = ce[_TITLE][_TEXT]
                    segment = _HTML

            if title and title != '':
                if _KNOWLEDGE_GRAPH not in doc:
                    doc[_KNOWLEDGE_GRAPH] = dict()
                doc[_KNOWLEDGE_GRAPH][_TITLE] = list()
                o = dict()
                o['value'] = title
                o['key'] = 'title'
                o['confidence'] = 1
                o['provenance'] = [Core.custom_provenance_object(method, segment, doc[_DOCUMENT_ID])]
                doc[_KNOWLEDGE_GRAPH][_TITLE].append(o)

        return doc

    @staticmethod
    def extract_tld(url):
        return tldextract.extract(url).domain + '.' + tldextract.extract(url).suffix

    @staticmethod
    def create_knowledge_graph(doc, field_name, extractions):
        if _KNOWLEDGE_GRAPH not in doc:
            doc[_KNOWLEDGE_GRAPH] = dict()

        if field_name not in doc[_KNOWLEDGE_GRAPH]:
            doc[_KNOWLEDGE_GRAPH][field_name] = dict()

        for extraction in extractions:
            key = extraction['value']
            if (isinstance(key, basestring) or isinstance(key, numbers.Number)) and field_name != _POSTING_DATE:
                # try except block because unicode characters will not be lowered
                try:
                    key = str(key).strip().lower()
                except:
                    pass
            if 'metadata' in extraction:
                sorted_metadata = Core.sort_dict(extraction['metadata'])
                for k, v in sorted_metadata.iteritems():
                    if isinstance(v, numbers.Number):
                        v = str(v)
                    # if v:
                    #     v = v.encode('utf-8')
                    if v and v.strip() != '':
                        # key += '-' + str(k) + ':' + str(v)
                        key = '{}-{}:{}'.format(key, k, v)

            if 'key' in extraction:
                key = extraction['key']

            # TODO FIX THIS HACK
            if len(key) > 32766:
                key = key[0:500]

            if key not in doc[_KNOWLEDGE_GRAPH][field_name]:
                doc[_KNOWLEDGE_GRAPH][field_name][key] = list()
            doc[_KNOWLEDGE_GRAPH][field_name][key].append(extraction)

        return doc

    @staticmethod
    def reformat_knowledge_graph(knowledge_graph):
        new_kg = dict()
        for semantic_type in knowledge_graph.keys():
            new_kg[semantic_type] = list()
            values = knowledge_graph[semantic_type]
            for key in values.keys():
                o = dict()
                o['key'] = key
                new_provenances, metadata, value = Core.rearrange_provenance(values[key])
                o['provenance'] = new_provenances
                if metadata:
                    o['qualifiers'] = metadata
                o['value'] = value
                # default confidence value, to be updated by later analysis
                o['confidence'] = 1
                new_kg[semantic_type].append(o)
        return new_kg

    @staticmethod
    def rearrange_provenance(old_provenances):
        new_provenances = list()
        metadata = None
        value = None
        for prov in old_provenances:
            new_prov = dict()
            method = None
            confidence = None

            if 'origin' in prov:
                origin = prov['origin']
                if 'obfuscation' in prov:
                    origin['extraction_metadata'] = dict()
                    origin['extraction_metadata']['obfuscation'] = prov['obfuscation']
                method = origin['method']
                confidence = origin['score']
                origin.pop('score', None)
                origin.pop('method', None)
                new_prov['source'] = origin

            if 'context' in prov:
                new_prov['source']['context'] = prov['context']
            if 'metadata' in prov and not metadata:
                metadata = prov['metadata']
            if method:
                new_prov["method"] = method
            if not value:
                value = prov['value']
            new_prov['extracted_value'] = value
            if confidence:
                new_prov['confidence'] = dict()
                new_prov['confidence']['extraction'] = confidence
            new_provenances.append(new_prov)
        return new_provenances, metadata, value

    @staticmethod
    def add_data_extraction_results(d, field_name, method_name, results):
        if _DATA_EXTRACTION not in d:
            d[_DATA_EXTRACTION] = dict()
        if field_name not in d[_DATA_EXTRACTION]:
            d[_DATA_EXTRACTION][field_name] = dict()
        if method_name not in d[_DATA_EXTRACTION][field_name]:
            d[_DATA_EXTRACTION][field_name][method_name] = dict()
        if isinstance(results, dict):
            results = [results]
        if 'results' not in d[_DATA_EXTRACTION][field_name][method_name]:
            d[_DATA_EXTRACTION][field_name][method_name]['results'] = results
        else:
            d[_DATA_EXTRACTION][field_name][method_name]['results'].extend(results)
        return d

    @staticmethod
    def check_if_run_extraction(d, field_name, method_name, extraction_policy):
        if _DATA_EXTRACTION not in d:
            return True
        if field_name not in d[_DATA_EXTRACTION]:
            return True
        if method_name not in d[_DATA_EXTRACTION][field_name]:
            return True
        if 'results' not in d[_DATA_EXTRACTION][field_name][method_name]:
            return True
        else:
            if extraction_policy == _REPLACE:
                return True
        return False

    @staticmethod
    def determine_segment(json_path):
        segment = _SEGMENT_OTHER
        if _SEGMENT_INFERLINK_DESC in json_path:
            segment = _SEGMENT_INFERLINK_DESC
        elif _INFERLINK in json_path and _SEGMENT_INFERLINK_DESC not in json_path:
            segment = _HTML
        elif _CONTENT_STRICT in json_path:
            segment = _CONTENT_STRICT
        elif _CONTENT_RELAXED in json_path:
            segment = _CONTENT_RELAXED
        elif _TITLE in json_path:
            segment = _TITLE
        elif _URL in json_path:
            segment = _URL
        return segment

    @staticmethod
    def add_origin_info(results, method, segment, score, doc_id=None):
        if results:
            for result in results:
                o = dict()
                o['segment'] = segment
                o['method'] = method
                o['score'] = score
                if doc_id:
                    o[_DOCUMENT_ID] = doc_id
                result['origin'] = o
        return results

    def run_landmark(self, content_extraction, html, landmark_config, url):
        field_name = landmark_config[_FIELD_NAME] if _FIELD_NAME in landmark_config else _INFERLINK_EXTRACTIONS
        ep = self.determine_extraction_policy(landmark_config)
        extraction_rules = self.consolidate_landmark_rules()
        if _LANDMARK_THRESHOLD in landmark_config:
            pct = landmark_config[_LANDMARK_THRESHOLD]
            if not 0.0 <= pct <= 1.0:
                raise ValueError('landmark threshold should be a float between {} and {}'.format(0.0, 1.0))
        else:
            pct = 0.5
        if field_name not in content_extraction or (field_name in content_extraction and ep == _REPLACE):
            start_time = time.time()
            ifl_extractions = Core.extract_landmark(html, url, extraction_rules, pct)

            if isinstance(ifl_extractions, list):
                # we have a rogue post type page, put it in its place
                content_extraction[field_name] = dict()
                content_extraction[field_name]['inferlink_posts'] = ifl_extractions
            else:
                time_taken = time.time() - start_time
                if self.debug:
                    print 'time taken to process landmark %s' % time_taken
                if ifl_extractions and len(ifl_extractions.keys()) > 0:
                    content_extraction[field_name] = dict()
                    for key in ifl_extractions:
                        o = dict()
                        o[key] = dict()
                        o[key]['text'] = ifl_extractions[key]
                        content_extraction[field_name].update(o)
        return content_extraction

    def consolidate_landmark_rules(self):
        rules = dict()
        if _RESOURCES in self.extraction_config:
            resources = self.extraction_config[_RESOURCES]
            if _LANDMARK in resources:
                landmark_rules_file_list = resources[_LANDMARK]
                for landmark_rules_file in landmark_rules_file_list:
                    rules.update(Core.load_json_file(landmark_rules_file))
                return rules
            else:
                raise KeyError('{}.{} not found in provided extraction config'.format(_RESOURCES, _LANDMARK))
        else:
            raise KeyError('{} not found in provided extraction config'.format(_RESOURCES))

    def get_dict_file_name_from_config(self, dict_name):
        if _RESOURCES in self.extraction_config:
            resources = self.extraction_config[_RESOURCES]
            if _DICTIONARIES in resources:
                if dict_name in resources[_DICTIONARIES]:
                    return resources[_DICTIONARIES][dict_name]
                else:
                    raise KeyError(
                        '{}.{}.{} not found in provided extraction config'.format(_RESOURCES, _DICTIONARIES, dict_name))
            else:
                raise KeyError('{}.{} not found in provided extraction config'.format(_RESOURCES, _DICTIONARIES))
        else:
            raise KeyError('{} not found in provided extraction config'.format(_RESOURCES))
<commit_msg>posts to text<commit_after>import sys

stdout = sys.stdout
reload(sys)
sys.setdefaultencoding('utf-8')
sys.stdout = stdout
# import all extractors
from spacy_extractors import age_extractor as spacy_age_extractor
from spacy_extractors import social_media_extractor as spacy_social_media_extractor
from spacy_extractors import date_extractor as spacy_date_extractor
from spacy_extractors import address_extractor as spacy_address_extractor
from spacy_extractors import customized_extractor as custom_spacy_extractor
from data_extractors import landmark_extraction
from data_extractors import dictionary_extractor
from data_extractors import regex_extractor
from data_extractors import height_extractor
from data_extractors import weight_extractor
from data_extractors import address_extractor
from data_extractors import age_extractor
from data_extractors import table_extractor
from data_extractors import url_country_extractor
from data_extractors import geonames_extractor
from data_extractors.digPhoneExtractor import phone_extractor
from data_extractors.digEmailExtractor import email_extractor
from data_extractors.digPriceExtractor import price_extractor
from data_extractors.digReviewIDExtractor import review_id_extractor
from data_extractors import date_parser
from classifiers import country_classifier
from structured_extractors import ReadabilityExtractor, TokenizerExtractor, FaithfulTokenizerExtractor
import json
import gzip
import re
import spacy
import codecs
from jsonpath_rw import parse
import time
import collections
import numbers
from tldextract import tldextract
import pickle
import copy
from collections import OrderedDict
import sys

_KNOWLEDGE_GRAPH = "knowledge_graph"
_EXTRACTION_POLICY = 'extraction_policy'
_KEEP_EXISTING = 'keep_existing'
_REPLACE = 'replace'
_ERROR_HANDLING = 'error_handling'
_IGNORE_EXTRACTION = 'ignore_extraction'
_IGNORE_DOCUMENT = 'ignore_document'
_RAISE_ERROR = 'raise_error'
_CITY_NAME = 'city_name'
_STATE = 'state'
_COUNTRY = 'country'
_CONTENT_EXTRACTION = 'content_extraction'
_SPACY_EXTRACTION = 'spacy_extraction'
_RAW_CONTENT = 'raw_content'
_INPUT_PATH = 'input_path'
_READABILITY = 'readability'
_LANDMARK = 'landmark'
_TITLE = 'title'
_DESCRIPTION = "description"
_STRICT = 'strict'
_FIELD_NAME = 'field_name'
_CONTENT_STRICT = 'content_strict'
_CONTENT_RELAXED = 'content_relaxed'
_YES = 'yes'
_NO = 'no'
_RECALL_PRIORITY = 'recall_priority'
_INFERLINK_EXTRACTIONS = 'inferlink_extractions'
_LANDMARK_THRESHOLD = 'landmark_threshold'
_LANDMARK_RULES = 'landmark_rules'
_URL = 'url'
_AGE = 'age'
_POSTING_DATE = 'posting_date'
_SOCIAL_MEDIA = 'social_media'
_ADDRESS = 'address'
_RESOURCES = 'resources'
_SPACY_FIELD_RULES = "spacy_field_rules"
_DATA_EXTRACTION = 'data_extraction'
_FIELDS = 'fields'
_EXTRACTORS = 'extractors'
_TOKENS = 'tokens'
_TOKENS_ORIGINAL_CASE = "tokens_original_case"
_SIMPLE_TOKENS = 'simple_tokens'
_SIMPLE_TOKENS_ORIGINAL_CASE = 'simple_tokens_original_case'
_TEXT = 'text'
_DICTIONARY = 'dictionary'
_PICKLES = 'pickle'
_NGRAMS = 'ngrams'
_JOINER = 'joiner'
_PRE_FILTER = 'pre_filter'
_POST_FILTER = 'post_filter'
_PRE_PROCESS = "pre_process"
_TABLE = "table"
_STOP_WORDS = "stop_words"
_GEONAMES = "geonames"
_STATE_TO_COUNTRY = "state_to_country"
_STATE_TO_CODES_LOWER = "state_to_codes_lower"
_POPULATED_PLACES = "populated_places"
_POPULATED_CITIES = "populated_cities"
_CASE_SENSITIVE = 'case_sensitive'

_EXTRACT_USING_DICTIONARY = "extract_using_dictionary"
_EXTRACT_USING_REGEX = "extract_using_regex"
_EXTRACT_FROM_LANDMARK = "extract_from_landmark"
_EXTRACT_PHONE = "extract_phone"
_EXTRACT_EMAIL = "extract_email"
_EXTRACT_PRICE = "extract_price"
_EXTRACT_HEIGHT = "extract_height"
_EXTRACT_WEIGHT = "extract_weight"
_EXTRACT_ADDRESS = "extract_address"
_EXTRACT_AGE = "extract_age"

_CONFIG = "config"
_DICTIONARIES = "dictionaries"
_INFERLINK = "inferlink"
_HTML = "html"

_SEGMENT_TITLE = "title"
_SEGMENT_INFERLINK_DESC = "inferlink_description"
_SEGMENT_OTHER = "other_segment"

_METHOD_INFERLINK = "inferlink"

_SOURCE_TYPE = "source_type"
_OBFUSCATION = "obfuscation"

_INCLUDE_CONTEXT = "include_context"
_KG_ENHANCEMENT = "kg_enhancement"
_DOCUMENT_ID = "document_id"
_TLD = 'tld'
_FEATURE_COMPUTATION = "feature_computation"


class Core(object):
    def __init__(self, extraction_config=None, debug=False, load_spacy=False):
        self.extraction_config = extraction_config
        self.debug = debug
        self.html_title_regex = r'<title>(.*?)</title>'
        self.tries = dict()
        self.pickles = dict()
        self.jobjs = dict()
        self.global_extraction_policy = None
        self.global_error_handling = None
        # to make sure we do not parse json_paths more times than needed, we define the following 2 properties
        self.content_extraction_path = None
        self.data_extraction_path = dict()
        if load_spacy:
            self.prep_spacy()
        else:
            self.nlp = None
        self.country_code_dict = None
        self.matchers = dict()
        self.geonames_dict = None
        self.state_to_country_dict = None
        self.state_to_codes_lower_dict = None
        self.populated_cities = None

    """ Define all API methods """

    def process(self, doc, create_knowledge_graph=False):
        try:
            # print 'Now Processing url: {}, doc_id: {}'.format(doc['url'], doc['doc_id'])
            if self.extraction_config:
                doc_id = None
                if _DOCUMENT_ID in self.extraction_config:
                    doc_id_field = self.extraction_config[_DOCUMENT_ID]
                    if doc_id_field in doc:
                        doc_id = doc[doc_id_field]
                        doc[_DOCUMENT_ID] = doc_id
                    else:
                        raise KeyError('{} not found in the input document'.format(doc_id_field))
                if _EXTRACTION_POLICY in self.extraction_config:
                    self.global_extraction_policy = self.extraction_config[_EXTRACTION_POLICY]
                if _ERROR_HANDLING in self.extraction_config:
                    self.global_error_handling = self.extraction_config[_ERROR_HANDLING]

                """Handle content extraction first aka Phase 1"""
                if _CONTENT_EXTRACTION in self.extraction_config:
                    if _CONTENT_EXTRACTION not in doc:
                        doc[_CONTENT_EXTRACTION] = dict()
                    ce_config = self.extraction_config[_CONTENT_EXTRACTION]
                    html_path = ce_config[_INPUT_PATH] if _INPUT_PATH in ce_config else None
                    if not html_path:
                        raise KeyError('{} not found in extraction_config'.format(_INPUT_PATH))

                    if not self.content_extraction_path:
                        start_time = time.time()
                        self.content_extraction_path = parse(html_path)
                        time_taken = time.time() - start_time
                        if self.debug:
                            print 'time taken to process parse %s' % time_taken
                    start_time = time.time()
                    matches = self.content_extraction_path.find(doc)
                    time_taken = time.time() - start_time
                    if self.debug:
                        print 'time taken to process matches %s' % time_taken
                    extractors = ce_config[_EXTRACTORS]
                    for index in range(len(matches)):
                        for extractor in extractors.keys():
                            if extractor == _READABILITY:
                                # TODO REMOVE THIS HACK
                                if len(matches[index].value) < 700000 and 'amigobulls.com' not in doc['url'] and 'kulakowka.com' not in doc['url']:
                                    re_extractors = extractors[extractor]
                                    if isinstance(re_extractors, dict):
                                        re_extractors = [re_extractors]
                                    for re_extractor in re_extractors:
                                        doc[_CONTENT_EXTRACTION] = self.run_readability(doc[_CONTENT_EXTRACTION],
                                                                                        matches[index].value,
                                                                                        re_extractor)
                                else:
                                    print 'Large document not running READABILITY, doc_id: {}'.format(doc['doc_id'])
                            elif extractor == _TITLE:
                                doc[_CONTENT_EXTRACTION] = self.run_title(doc[_CONTENT_EXTRACTION],
                                                                          matches[index].value,
                                                                          extractors[extractor])
                            elif extractor == _LANDMARK:
                                doc[_CONTENT_EXTRACTION] = self.run_landmark(doc[_CONTENT_EXTRACTION],
                                                                             matches[index].value,
                                                                             extractors[extractor], doc[_URL])
                            elif extractor == _TABLE:
                                doc[_CONTENT_EXTRACTION] = self.run_table_extractor(doc[_CONTENT_EXTRACTION],
                                                                                    matches[index].value,
                                                                                    extractors[extractor])
                    # Add the url as segment as well
                    if _URL in doc and doc[_URL] and doc[_URL].strip() != '':
                        doc[_CONTENT_EXTRACTION][_URL] = dict()
                        doc[_CONTENT_EXTRACTION][_URL][_TEXT] = doc[_URL]
                        doc[_TLD] = self.extract_tld(doc[_URL])

                """Phase 2: The Data Extraction"""
                if _DATA_EXTRACTION in self.extraction_config:
                    de_configs = self.extraction_config[_DATA_EXTRACTION]
                    if isinstance(de_configs, dict):
                        de_configs = [de_configs]

                    for i in range(len(de_configs)):
                        de_config = de_configs[i]
                        input_paths = de_config[_INPUT_PATH] if _INPUT_PATH in de_config else None
                        if not input_paths:
                            raise KeyError('{} not found for data extraction in extraction_config'.format(_INPUT_PATH))

                        if not isinstance(input_paths, list):
                            input_paths = [input_paths]

                        for input_path in input_paths:
                            if _FIELDS in de_config:
                                if input_path not in self.data_extraction_path:
                                    self.data_extraction_path[input_path] = parse(input_path)
                                matches = self.data_extraction_path[input_path].find(doc)
                                for match in matches:
                                    # First rule of DATA Extraction club: Get tokens
                                    # Get the crf tokens
                                    if _TEXT in match.value:
                                        if _TOKENS_ORIGINAL_CASE not in match.value:
                                            match.value[_TOKENS_ORIGINAL_CASE] = self.extract_crftokens(
                                                match.value[_TEXT],
                                                lowercase=False)
                                        if _TOKENS not in match.value:
                                            match.value[_TOKENS] = self.crftokens_to_lower(
                                                match.value[_TOKENS_ORIGINAL_CASE])
                                        if _SIMPLE_TOKENS not in match.value:
                                            match.value[_SIMPLE_TOKENS] = self.extract_tokens_from_crf(
                                                match.value[_TOKENS])
                                        if _SIMPLE_TOKENS_ORIGINAL_CASE not in match.value:
                                            match.value[_SIMPLE_TOKENS_ORIGINAL_CASE] = self.extract_tokens_from_crf(
                                                match.value[_TOKENS_ORIGINAL_CASE])
                                            # if _TOKENS not in match.value:
                                            #     match.value[_TOKENS] = self.extract_crftokens(match.value[_TEXT])
                                            # if _SIMPLE_TOKENS not in match.value:
                                            #     match.value[_SIMPLE_TOKENS] = self.extract_tokens_from_crf(match.value[_TOKENS])
                                    fields = de_config[_FIELDS]
                                    for field in fields.keys():
                                        if field != '*':
                                            """
                                                Special case for inferlink extractions:
                                                For eg, We do not want to extract name from inferlink_posting-date #DUH
                                            """
                                            run_extractor = True
                                            full_path = str(match.full_path)
                                            segment = self.determine_segment(full_path)
                                            if _INFERLINK in full_path:
                                                if field not in full_path:
                                                    run_extractor = False
                                                if _DESCRIPTION in full_path or _TITLE in full_path:
                                                    run_extractor = True
                                            if run_extractor:
                                                if _EXTRACTORS in fields[field]:
                                                    extractors = fields[field][_EXTRACTORS]
                                                    for extractor in extractors.keys():
                                                        try:
                                                            foo = getattr(self, extractor)
                                                        except:
                                                            foo = None
                                                        if foo:
                                                            # score is 1.0 because every method thinks it is the best
                                                            score = 1.0
                                                            method = extractor
                                                            if _CONFIG not in extractors[extractor]:
                                                                extractors[extractor][_CONFIG] = dict()
                                                            extractors[extractor][_CONFIG][_FIELD_NAME] = field
                                                            ep = self.determine_extraction_policy(extractors[extractor])
                                                            if extractor == _EXTRACT_FROM_LANDMARK:
                                                                if _INFERLINK_EXTRACTIONS in full_path and field in full_path:
                                                                    method = _METHOD_INFERLINK
                                                                    if self.check_if_run_extraction(match.value, field,
                                                                                                    extractor,
                                                                                                    ep):

                                                                        results = foo(doc,
                                                                                      extractors[extractor][_CONFIG])
                                                                        if results:
                                                                            self.add_data_extraction_results(
                                                                                match.value,
                                                                                field,
                                                                                extractor,
                                                                                self.add_origin_info(
                                                                                    results,
                                                                                    method,
                                                                                    segment,
                                                                                    score,
                                                                                    doc_id))
                                                                            if create_knowledge_graph:
                                                                                self.create_knowledge_graph(doc, field,
                                                                                                            results)
                                                            else:
                                                                if self.check_if_run_extraction(match.value, field,
                                                                                                extractor,
                                                                                                ep):
                                                                    results = foo(match.value,
                                                                                  extractors[extractor][_CONFIG])
                                                                    if results:
                                                                        self.add_data_extraction_results(match.value,
                                                                                                         field,
                                                                                                         extractor,
                                                                                                         self.add_origin_info(
                                                                                                             results,
                                                                                                             method,
                                                                                                             segment,
                                                                                                             score,
                                                                                                             doc_id))
                                                                        if create_knowledge_graph:
                                                                            self.create_knowledge_graph(doc, field,
                                                                                                        results)
                                        else:  # extract whatever you can!
                                            if _EXTRACTORS in fields[field]:
                                                extractors = fields[field][_EXTRACTORS]
                                                for extractor in extractors.keys():
                                                    try:
                                                        foo = getattr(self, extractor)
                                                    except Exception as e:
                                                        foo = None
                                                    if foo:
                                                        # score is 1.0 because every method thinks it is the best
                                                        score = 1.0
                                                        method = extractor
                                                        if _CONFIG not in extractors[extractor]:
                                                            extractors[extractor][_CONFIG] = dict()
                                                        ep = self.determine_extraction_policy(extractors[extractor])
                                                        if extractor == _EXTRACT_FROM_LANDMARK:
                                                            if _INFERLINK_EXTRACTIONS in full_path and field in full_path:
                                                                method = _METHOD_INFERLINK
                                                                if self.check_if_run_extraction(match.value, field,
                                                                                                extractor,
                                                                                                ep):

                                                                    results = foo(doc, extractors[extractor][_CONFIG])
                                                                    if results:
                                                                        self.add_data_extraction_results(match.value,
                                                                                                         field,
                                                                                                         extractor,
                                                                                                         self.add_origin_info(
                                                                                                             results,
                                                                                                             method,
                                                                                                             segment,
                                                                                                             score,
                                                                                                             doc_id))
                                                                        if create_knowledge_graph:
                                                                            self.create_knowledge_graph(doc, field,
                                                                                                        results)
                                                        else:
                                                            results = foo(match.value,
                                                                          extractors[extractor][_CONFIG])
                                                            if results:
                                                                for f, res in results.items():
                                                                    # extractors[extractor][_CONFIG][_FIELD_NAME] = f
                                                                    self.add_data_extraction_results(match.value, f,
                                                                                                     extractor,
                                                                                                     self.add_origin_info(
                                                                                                         res,
                                                                                                         method,
                                                                                                         segment,
                                                                                                         score, doc_id))
                                                                    if create_knowledge_graph:
                                                                        self.create_knowledge_graph(doc, f, res)
                                                    else:
                                                        print('method {} not found!'.format(extractor))

                """Optional Phase 3: Knowledge Graph Enhancement"""
                if _KG_ENHANCEMENT in self.extraction_config:
                    kg_configs = self.extraction_config[_KG_ENHANCEMENT]
                    if isinstance(kg_configs, dict):
                        kg_configs = [kg_configs]

                    for i in range(len(kg_configs)):
                        kg_config = kg_configs[i]
                        input_paths = kg_config[_INPUT_PATH] if _INPUT_PATH in kg_config else None
                        if not input_paths:
                            raise KeyError(
                                '{} not found for knowledge graph enhancement in extraction_config'.format(_INPUT_PATH))

                        if not isinstance(input_paths, list):
                            input_paths = [input_paths]

                        for input_path in input_paths:
                            if _FIELDS in kg_config:
                                if input_path not in self.data_extraction_path:
                                    self.data_extraction_path[input_path] = parse(input_path)
                                matches = self.data_extraction_path[input_path].find(doc)
                                for match in matches:
                                    fields = kg_config[_FIELDS]
                                    try:
                                        sorted_fields = self.sort_dictionary_by_fields(fields)
                                    except:
                                        raise ValueError('Please ensure there is a priority added to every field in '
                                                         'knowledge_graph  enhancement and the priority is an int')
                                    for i in range(0, len(sorted_fields)):
                                        field = sorted_fields[i][0]
                                        if _EXTRACTORS in fields[field]:
                                            extractors = fields[field][_EXTRACTORS]
                                            for extractor in extractors.keys():
                                                try:
                                                    foo = getattr(self, extractor)
                                                except:
                                                    foo = None
                                                if foo:
                                                    if _CONFIG not in extractors[extractor]:
                                                        extractors[extractor][_CONFIG] = dict()
                                                    extractors[extractor][_CONFIG][_FIELD_NAME] = field
                                                    results = foo(match.value, extractors[extractor][_CONFIG])
                                                    if results:
                                                        # doc[_KNOWLEDGE_GRA][field] = results
                                                        self.create_knowledge_graph(doc, field, results)

                """Optional Phase 4: feature computation"""
                if _FEATURE_COMPUTATION in self.extraction_config:
                    kg_configs = self.extraction_config[_FEATURE_COMPUTATION]
                    if isinstance(kg_configs, dict):
                        kg_configs = [kg_configs]

                    for i in range(len(kg_configs)):
                        kg_config = kg_configs[i]
                        input_paths = kg_config[_INPUT_PATH] if _INPUT_PATH in kg_config else None
                        if not input_paths:
                            raise KeyError(
                                '{} not found for feature computation in extraction_config'.format(_INPUT_PATH))

                        if not isinstance(input_paths, list):
                            input_paths = [input_paths]

                        for input_path in input_paths:
                            if _FIELDS in kg_config:
                                if input_path not in self.data_extraction_path:
                                    self.data_extraction_path[input_path] = parse(input_path)
                                matches = self.data_extraction_path[input_path].find(doc)
                                for match in matches:
                                    fields = kg_config[_FIELDS]
                                    for field in fields.keys():
                                        if _EXTRACTORS in fields[field]:
                                            extractors = fields[field][_EXTRACTORS]
                                            for extractor in extractors.keys():
                                                try:
                                                    foo = getattr(self, extractor)
                                                except:
                                                    foo = None
                                                if foo:
                                                    if _CONFIG not in extractors[extractor]:
                                                        extractors[extractor][_CONFIG] = dict()
                                                    extractors[extractor][_CONFIG][_FIELD_NAME] = field
                                                    results = foo(match.value, extractors[extractor][_CONFIG])
                                                    if results:
                                                        # doc[_KNOWLEDGE_GRAPH][field] = results
                                                        self.create_knowledge_graph(doc, field, results)

                if _KNOWLEDGE_GRAPH in doc and doc[_KNOWLEDGE_GRAPH]:
                    doc[_KNOWLEDGE_GRAPH] = self.reformat_knowledge_graph(doc[_KNOWLEDGE_GRAPH])
                    """ Add title and description as fields in the knowledge graph as well"""
                    doc = Core.rearrange_description(doc)
                    doc = Core.rearrange_title(doc)
        except Exception as e:
            print e
            print 'Failed doc:', doc['doc_id']
            raise e
        # print 'DONE url: {}, doc_id: {}'.format(doc['url'], doc['doc_id'])
        return doc

    @staticmethod
    def rearrange_description(doc):
        method = 'rearrange_description'
        description = None
        segment = ''
        if _CONTENT_EXTRACTION in doc:
            ce = doc[_CONTENT_EXTRACTION]
            if _INFERLINK_EXTRACTIONS in ce:
                if _DESCRIPTION in ce[_INFERLINK_EXTRACTIONS]:
                    description = ce[_INFERLINK_EXTRACTIONS][_DESCRIPTION][_TEXT]
                    segment = _INFERLINK
            if not description or description.strip() == '':
                if _CONTENT_STRICT in ce:
                    description = ce[_CONTENT_STRICT][_TEXT]
                    segment = _CONTENT_STRICT

            if description and description != '':
                if _KNOWLEDGE_GRAPH not in doc:
                    doc[_KNOWLEDGE_GRAPH] = dict()
                doc[_KNOWLEDGE_GRAPH][_DESCRIPTION] = list()
                o = dict()
                o['value'] = description
                o['key'] = 'description'
                o['confidence'] = 1
                o['provenance'] = [Core.custom_provenance_object(method, segment, doc[_DOCUMENT_ID])]
                doc[_KNOWLEDGE_GRAPH][_DESCRIPTION].append(o)
        return doc

    @staticmethod
    def sort_dictionary_by_fields(dictionary):
        sorted_d = OrderedDict(sorted(dictionary.iteritems(), key=lambda x: x[1]['priority']))
        return sorted_d.items()

    @staticmethod
    def custom_provenance_object(method, segment, document_id):
        prov = dict()
        prov['method'] = method
        prov['source'] = dict()
        prov['source']['segment'] = segment
        prov['source'][_DOCUMENT_ID] = document_id
        return prov

    @staticmethod
    def rearrange_title(doc):
        method = 'rearrange_title'
        title = None
        segment = ''
        if _CONTENT_EXTRACTION in doc:
            ce = doc[_CONTENT_EXTRACTION]
            if _INFERLINK_EXTRACTIONS in ce:
                if _TITLE in ce[_INFERLINK_EXTRACTIONS]:
                    title = ce[_INFERLINK_EXTRACTIONS][_TITLE][_TEXT]
                    segment = _INFERLINK
            if not title or title.strip() == '':
                if _TITLE in ce:
                    title = ce[_TITLE][_TEXT]
                    segment = _HTML

            if title and title != '':
                if _KNOWLEDGE_GRAPH not in doc:
                    doc[_KNOWLEDGE_GRAPH] = dict()
                doc[_KNOWLEDGE_GRAPH][_TITLE] = list()
                o = dict()
                o['value'] = title
                o['key'] = 'title'
                o['confidence'] = 1
                o['provenance'] = [Core.custom_provenance_object(method, segment, doc[_DOCUMENT_ID])]
                doc[_KNOWLEDGE_GRAPH][_TITLE].append(o)

        return doc

    @staticmethod
    def extract_tld(url):
        return tldextract.extract(url).domain + '.' + tldextract.extract(url).suffix

    @staticmethod
    def create_knowledge_graph(doc, field_name, extractions):
        if _KNOWLEDGE_GRAPH not in doc:
            doc[_KNOWLEDGE_GRAPH] = dict()

        if field_name not in doc[_KNOWLEDGE_GRAPH]:
            doc[_KNOWLEDGE_GRAPH][field_name] = dict()

        for extraction in extractions:
            key = extraction['value']
            if (isinstance(key, basestring) or isinstance(key, numbers.Number)) and field_name != _POSTING_DATE:
                # try except block because unicode characters will not be lowered
                try:
                    key = str(key).strip().lower()
                except:
                    pass
            if 'metadata' in extraction:
                sorted_metadata = Core.sort_dict(extraction['metadata'])
                for k, v in sorted_metadata.iteritems():
                    if isinstance(v, numbers.Number):
                        v = str(v)
                    # if v:
                    #     v = v.encode('utf-8')
                    if v and v.strip() != '':
                        # key += '-' + str(k) + ':' + str(v)
                        key = '{}-{}:{}'.format(key, k, v)

            if 'key' in extraction:
                key = extraction['key']

            # TODO FIX THIS HACK
            if len(key) > 32766:
                key = key[0:500]

            if key not in doc[_KNOWLEDGE_GRAPH][field_name]:
                doc[_KNOWLEDGE_GRAPH][field_name][key] = list()
            doc[_KNOWLEDGE_GRAPH][field_name][key].append(extraction)

        return doc

    @staticmethod
    def reformat_knowledge_graph(knowledge_graph):
        new_kg = dict()
        for semantic_type in knowledge_graph.keys():
            new_kg[semantic_type] = list()
            values = knowledge_graph[semantic_type]
            for key in values.keys():
                o = dict()
                o['key'] = key
                new_provenances, metadata, value = Core.rearrange_provenance(values[key])
                o['provenance'] = new_provenances
                if metadata:
                    o['qualifiers'] = metadata
                o['value'] = value
                # default confidence value, to be updated by later analysis
                o['confidence'] = 1
                new_kg[semantic_type].append(o)
        return new_kg

    @staticmethod
    def rearrange_provenance(old_provenances):
        new_provenances = list()
        metadata = None
        value = None
        for prov in old_provenances:
            new_prov = dict()
            method = None
            confidence = None

            if 'origin' in prov:
                origin = prov['origin']
                if 'obfuscation' in prov:
                    origin['extraction_metadata'] = dict()
                    origin['extraction_metadata']['obfuscation'] = prov['obfuscation']
                method = origin['method']
                confidence = origin['score']
                origin.pop('score', None)
                origin.pop('method', None)
                new_prov['source'] = origin

            if 'context' in prov:
                new_prov['source']['context'] = prov['context']
            if 'metadata' in prov and not metadata:
                metadata = prov['metadata']
            if method:
                new_prov["method"] = method
            if not value:
                value = prov['value']
            new_prov['extracted_value'] = value
            if confidence:
                new_prov['confidence'] = dict()
                new_prov['confidence']['extraction'] = confidence
            new_provenances.append(new_prov)
        return new_provenances, metadata, value

    @staticmethod
    def add_data_extraction_results(d, field_name, method_name, results):
        if _DATA_EXTRACTION not in d:
            d[_DATA_EXTRACTION] = dict()
        if field_name not in d[_DATA_EXTRACTION]:
            d[_DATA_EXTRACTION][field_name] = dict()
        if method_name not in d[_DATA_EXTRACTION][field_name]:
            d[_DATA_EXTRACTION][field_name][method_name] = dict()
        if isinstance(results, dict):
            results = [results]
        if 'results' not in d[_DATA_EXTRACTION][field_name][method_name]:
            d[_DATA_EXTRACTION][field_name][method_name]['results'] = results
        else:
            d[_DATA_EXTRACTION][field_name][method_name]['results'].extend(results)
        return d

    @staticmethod
    def check_if_run_extraction(d, field_name, method_name, extraction_policy):
        if _DATA_EXTRACTION not in d:
            return True
        if field_name not in d[_DATA_EXTRACTION]:
            return True
        if method_name not in d[_DATA_EXTRACTION][field_name]:
            return True
        if 'results' not in d[_DATA_EXTRACTION][field_name][method_name]:
            return True
        else:
            if extraction_policy == _REPLACE:
                return True
        return False

    @staticmethod
    def determine_segment(json_path):
        segment = _SEGMENT_OTHER
        if _SEGMENT_INFERLINK_DESC in json_path:
            segment = _SEGMENT_INFERLINK_DESC
        elif _INFERLINK in json_path and _SEGMENT_INFERLINK_DESC not in json_path:
            segment = _HTML
        elif _CONTENT_STRICT in json_path:
            segment = _CONTENT_STRICT
        elif _CONTENT_RELAXED in json_path:
            segment = _CONTENT_RELAXED
        elif _TITLE in json_path:
            segment = _TITLE
        elif _URL in json_path:
            segment = _URL
        return segment

    @staticmethod
    def add_origin_info(results, method, segment, score, doc_id=None):
        if results:
            for result in results:
                o = dict()
                o['segment'] = segment
                o['method'] = method
                o['score'] = score
                if doc_id:
                    o[_DOCUMENT_ID] = doc_id
                result['origin'] = o
        return results

    def run_landmark(self, content_extraction, html, landmark_config, url):
        field_name = landmark_config[_FIELD_NAME] if _FIELD_NAME in landmark_config else _INFERLINK_EXTRACTIONS
        ep = self.determine_extraction_policy(landmark_config)
        extraction_rules = self.consolidate_landmark_rules()
        if _LANDMARK_THRESHOLD in landmark_config:
            pct = landmark_config[_LANDMARK_THRESHOLD]
            if not 0.0 <= pct <= 1.0:
                raise ValueError('landmark threshold should be a float between {} and {}'.format(0.0, 1.0))
        else:
            pct = 0.5
        if field_name not in content_extraction or (field_name in content_extraction and ep == _REPLACE):
            start_time = time.time()
            ifl_extractions = Core.extract_landmark(html, url, extraction_rules, pct)

            if isinstance(ifl_extractions, list):
                # we have a rogue post type page, put it in its place
                field_name = 'inferlink_posts_special_text'
                content_extraction[field_name] = dict()
                content_extraction[field_name][_TEXT] = self.inferlink_posts_to_text(ifl_extractions)
            else:
                time_taken = time.time() - start_time
                if self.debug:
                    print 'time taken to process landmark %s' % time_taken
                if ifl_extractions and len(ifl_extractions.keys()) > 0:
                    content_extraction[field_name] = dict()
                    for key in ifl_extractions:
                        o = dict()
                        o[key] = dict()
                        o[key]['text'] = ifl_extractions[key]
                        content_extraction[field_name].update(o)
        return content_extraction

    @staticmethod
    def inferlink_posts_to_text(inferlink_posts):
        text = ''
        for inferlink_post in inferlink_posts:
            for k, v in inferlink_post.iteritems():
                if k == 'review_details':
                    for pair in v:
                        if 'key' in pair and 'value' in pair:
                            text += '{}_{}: {}\n'.format('review_details', pair['key'], pair['value'])
                else:
                    text += '{}: {}\n'.format(k, v)
        return text

    def consolidate_landmark_rules(self):
        rules = dict()
        if _RESOURCES in self.extraction_config:
            resources = self.extraction_config[_RESOURCES]
            if _LANDMARK in resources:
                landmark_rules_file_list = resources[_LANDMARK]
                for landmark_rules_file in landmark_rules_file_list:
                    rules.update(Core.load_json_file(landmark_rules_file))
                return rules
            else:
                raise KeyError('{}.{} not found in provided extraction config'.format(_RESOURCES, _LANDMARK))
        else:
            raise KeyError('{} not found in provided extraction config'.format(_RESOURCES))

    def get_dict_file_name_from_config(self, dict_name):
        if _RESOURCES in self.extraction_config:
            resources = self.extraction_config[_RESOURCES]
            if _DICTIONARIES in resources:
                if dict_name in resources[_DICTIONARIES]:
                    return resources[_DICTIONARIES][dict_name]
                else:
                    raise KeyError(
                        '{}.{}.{} not found in provided extraction config'.format(_RESOURCES, _DICTIONARIES, dict_name))
            else:
                raise KeyError('{}.{} not found in provided extraction config'.format(_RESOURCES, _DICTIONARIES))
        else:
            raise KeyError('{} not found in provided extraction config'.format(_RESOURCES))

{'fraction_non_alphanumeric': 0.0584000368019137, 'fraction_numerical': 0.001081056214923176, 'mean_word_length': 1.0088019220995241, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1026, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2296864', 'n_tokens_mistral': 18794, 'n_tokens_neox': 17752, 'n_words': 4204}