<commit_before># -*- coding: utf-8 -*-
# @Author: ZwEin
# @Date:   2016-08-08 11:46:11
# @Last Modified by:   ZwEin
# @Last Modified time: 2016-08-11 13:35:50


import os
import csv
import json
import codecs
from node import *

DC_DATA_FILE_FORMAT_JSONLINES = 'jsonlines'
DC_DATA_FILE_FORMAT_JSON = 'json'
DC_DATA_FILE_FORMAT_CSV = 'csv'

DC_DATA_FILE_FORMATS = [
    DC_DATA_FILE_FORMAT_JSONLINES,
    DC_DATA_FILE_FORMAT_JSON,
    DC_DATA_FILE_FORMAT_CSV
]


DC_DEFAULT_DIG_WEBPAGE_DATA_FILEPATH = os.path.join(os.path.dirname(__file__), 'res', 'webpages.json')
DC_DEFAULT_TRAINING_DATA_FILEPATH = os.path.join(os.path.dirname(__file__), 'res', 'training_data.csv')
DC_DEFAULT_TESTING_DATA_FILEPATH = os.path.join(os.path.dirname(__file__), 'res', 'testing_data.csv')


class Loader(object):

    #################################################
    # Basic Load Data Functions
    #################################################

    def __load_data_jsonlines(path):
        # to be updated
        import jsonlines
        dataset = []
        lines = jsonlines.open(path, mode='r')
        for line in lines:
            dataset.append([line['sid'], line['content']])
        return dataset

    def __load_data_json(path):

        def es_content_loader(content):
            if not isinstance(content, basestring):
                content = ' '.join(content)
            return content

        dataset = []
        json_objs = json.load(codecs.open(path, 'r', 'utf-8'))
        for json_obj in json_objs:
            source = json_obj['_source']

            if DC_NODE_EXT_FEATURE_NAME_DOCID in source:
                doc_id = source[DC_NODE_EXT_FEATURE_NAME_DOCID]
            if raw_content in source:
                raw_content = source[DC_NODE_EXT_FEATURE_NAME_CONTENT]
            if extractions in source:
                extractions = source['extractions']

                if DC_NODE_EXT_FEATURE_NAME_POSTTIME in extractions:
                    posttime = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_POSTTIME]['results'])
                if DC_NODE_EXT_FEATURE_NAME_CITY in extractions:
                    city = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_CITY]['results'])
                if DC_NODE_EXT_FEATURE_NAME_TEXT in extractions:
                    text = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_TEXT]['results'])
                if DC_NODE_EXT_FEATURE_NAME_REGION in extractions:
                    region = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_REGION]['results'])
                if DC_NODE_EXT_FEATURE_NAME_TITLE in extractions:
                    title = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_TITLE]['results'])
                if DC_NODE_EXT_FEATURE_NAME_USERLOCATION in extractions:
                    userlocation = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_USERLOCATION]['results'])
                if DC_NODE_EXT_FEATURE_NAME_PHONENUMBER in extractions:
                    phonenumber = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_PHONENUMBER]['results'])
                if DC_NODE_EXT_FEATURE_NAME_SID in extractions:
                    sid = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_SID]['results'])
                if DC_NODE_EXT_FEATURE_NAME_OTHERADS in extractions:
                    otherads = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_OTHERADS]['results'])
                if DC_NODE_EXT_FEATURE_NAME_AGE in extractions:
                    age = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_AGE]['results'])
            


        return dataset

    def __load_data_csv(path):
        import csv
        dataset = []
        with open(path, 'rb') as csvfile:
        # with codecs.open(path, 'r', 'utf-8') as csvfile:
            reader = csv.reader(csvfile)
            for row in reader:
                new_node = Node(row[1].decode('utf-8', 'ignore').encode('ascii', 'ignore'), label=row[0], hasPhone=True)
                dataset.append(new_node)
                # break
        return dataset

    __loader_data_funcs = {<commit_msg>init variable<commit_after># -*- coding: utf-8 -*-
# @Author: ZwEin
# @Date:   2016-08-08 11:46:11
# @Last Modified by:   ZwEin
# @Last Modified time: 2016-08-11 13:38:14


import os
import csv
import json
import codecs
from node import *

DC_DATA_FILE_FORMAT_JSONLINES = 'jsonlines'
DC_DATA_FILE_FORMAT_JSON = 'json'
DC_DATA_FILE_FORMAT_CSV = 'csv'

DC_DATA_FILE_FORMATS = [
    DC_DATA_FILE_FORMAT_JSONLINES,
    DC_DATA_FILE_FORMAT_JSON,
    DC_DATA_FILE_FORMAT_CSV
]


DC_DEFAULT_DIG_WEBPAGE_DATA_FILEPATH = os.path.join(os.path.dirname(__file__), 'res', 'webpages.json')
DC_DEFAULT_TRAINING_DATA_FILEPATH = os.path.join(os.path.dirname(__file__), 'res', 'training_data.csv')
DC_DEFAULT_TESTING_DATA_FILEPATH = os.path.join(os.path.dirname(__file__), 'res', 'testing_data.csv')


class Loader(object):

    #################################################
    # Basic Load Data Functions
    #################################################

    def __load_data_jsonlines(path):
        # to be updated
        import jsonlines
        dataset = []
        lines = jsonlines.open(path, mode='r')
        for line in lines:
            dataset.append([line['sid'], line['content']])
        return dataset

    def __load_data_json(path):

        def es_content_loader(content):
            if not isinstance(content, basestring):
                content = ' '.join(content)
            return content

        dataset = []
        json_objs = json.load(codecs.open(path, 'r', 'utf-8'))
        for json_obj in json_objs:
            source = json_obj['_source']

            doc_id = raw_content = posttime = city = text = region = title = userlocation = phonenumber = sid = otherads = age = None

            if DC_NODE_EXT_FEATURE_NAME_DOCID in source:
                doc_id = source[DC_NODE_EXT_FEATURE_NAME_DOCID]
            if DC_NODE_EXT_FEATURE_NAME_CONTENT in source:
                raw_content = source[DC_NODE_EXT_FEATURE_NAME_CONTENT]
            if 'extractions' in source:
                extractions = source['extractions']

                if DC_NODE_EXT_FEATURE_NAME_POSTTIME in extractions:
                    posttime = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_POSTTIME]['results'])
                if DC_NODE_EXT_FEATURE_NAME_CITY in extractions:
                    city = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_CITY]['results'])
                if DC_NODE_EXT_FEATURE_NAME_TEXT in extractions:
                    text = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_TEXT]['results'])
                if DC_NODE_EXT_FEATURE_NAME_REGION in extractions:
                    region = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_REGION]['results'])
                if DC_NODE_EXT_FEATURE_NAME_TITLE in extractions:
                    title = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_TITLE]['results'])
                if DC_NODE_EXT_FEATURE_NAME_USERLOCATION in extractions:
                    userlocation = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_USERLOCATION]['results'])
                if DC_NODE_EXT_FEATURE_NAME_PHONENUMBER in extractions:
                    phonenumber = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_PHONENUMBER]['results'])
                if DC_NODE_EXT_FEATURE_NAME_SID in extractions:
                    sid = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_SID]['results'])
                if DC_NODE_EXT_FEATURE_NAME_OTHERADS in extractions:
                    otherads = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_OTHERADS]['results'])
                if DC_NODE_EXT_FEATURE_NAME_AGE in extractions:
                    age = es_content_loader(extractions[DC_NODE_EXT_FEATURE_NAME_AGE]['results'])
                
            


        return dataset

    def __load_data_csv(path):
        import csv
        dataset = []
        with open(path, 'rb') as csvfile:
        # with codecs.open(path, 'r', 'utf-8') as csvfile:
            reader = csv.reader(csvfile)
            for row in reader:
                new_node = Node(row[1].decode('utf-8', 'ignore').encode('ascii', 'ignore'), label=row[0], hasPhone=True)
                dataset.append(new_node)
                # break
        return dataset

    __loader_data_funcs = {
{'fraction_non_alphanumeric': 0.10662243809975122, 'fraction_numerical': 0.008055917545314536, 'mean_word_length': 2.804416403785489, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1947376', 'n_tokens_mistral': 2648, 'n_tokens_neox': 2533, 'n_words': 436}