<commit_before>#!/usr/bin/env python
"""
 AUTHOR: <NAME>
 DATE: 12-17-2013
 DEPENDENCIES: a list of modules requiring installation
 Copyright 2014 <NAME>

 LICENSE:
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.

 DESCRIPTION:
 Functions necessary to enrich the context graph

"""
# PRE-USER SETUP
from datetime import timedelta

########### NOT USER EDITABLE ABOVE THIS POINT #################


# USER VARIABLES
MAXMIND_FILE = "./GeoIPASNum.dat"
TITAN_HOST = "localhost"
TITAN_PORT = "8182"
TITAN_GRAPH = "vzgraph"
STATES = {'AA': 'armed forces americas', 'AE': 'armed forces middle east', 'AK': 'alaska', 'AL': 'alabama',
          'AP': 'armed forces pacific', 'AR': 'arkansas', 'AS': 'american samoa', 'AZ': 'arizona', 'CA': 'california',
          'CO': 'colorado', 'CT': 'connecticut', 'DC': 'district of columbia', 'DE': 'delaware', 'FL': 'florida',
          'FM': 'federated states of micronesia', 'GA': 'georgia', 'GU': 'guam', 'HI': 'hawaii', 'IA': 'iowa',
          'ID': 'idaho', 'IL': 'illinois', 'IN': 'indiana', 'KS': 'kansas', 'KY': 'kentucky', 'LA': 'louisiana',
          'MA': 'massachusetts', 'MD': 'maryland', 'ME': 'maine', 'MH': 'marshall islands', 'MI': 'michigan',
          'MN': 'minnesota', 'MO': 'missouri', 'MP': 'northern mariana islands', 'MS': 'mississippi', 'MT': 'montana',
          'NC': 'north carolina', 'ND': 'north dakota', 'NE': 'nebraska', 'NH': 'new hampshire', 'NJ': 'new jersey',
          'NM': 'new mexico', 'NV': 'nevada', 'NY': 'new york', 'OH': 'ohio', 'OK': 'oklahoma', 'OR': 'oregon',
          'PA': 'pennsylvania', 'PR': 'puerto rico', 'PW': 'palau', 'RI': 'rhode island', 'SC': 'south carolina',
          'SD': 'south dakota', 'TN': 'tennessee', 'TX': 'texas', 'UT': 'utah', 'VA': 'virginia',
          'VI': 'virgin islands', 'VT': 'vermont', 'WA': 'washington', 'WI': 'wisconsin', 'WV': 'west virginia',
          'WY': 'wyoming'}



########### NOT USER EDITABLE BELOW THIS POINT #################


## IMPORTS
import networkx as nx
import tldextract
import argparse
import logging
import sys
from datetime import datetime # timedelta imported above
import uuid
import copy
from cymru import CymruIPtoASNService
from cags_schema import TalksTo, DescribedBy, Influences
import GeoIP
from bulbs.titan import Graph as TITAN_Graph
from bulbs.titan import Config as TITAN_Config
#import csv
import ipaddress
import pandas as pd
from urlparse import urlparse
from collections import defaultdict
import socket
import tldextract
from bulbs.neo4jserver import NEO4J_URI
from bulbs.neo4jserver import Graph as NEO_Graph
from bulbs.neo4jserver import Config as NEO_Config
from py2neo import Graph as py2neoGraph
from py2neo import Relationship as py2neoRelationship
from py2neo import Node as py2neoNode

## SETUP
__author__ = "<NAME>"
# Parse Arguments (should correspond to user variables)
parser = argparse.ArgumentParser(description='This script processes a graph.')
parser.add_argument('-d', '--debug',
                    help='Print lots of debugging statements',
                    action="store_const", dest="loglevel", const=logging.DEBUG,
                    default=logging.WARNING
                   )
parser.add_argument('-v', '--verbose',
                    help='Be verbose',
                    action="store_const", dest="loglevel", const=logging.INFO
                   )
parser.add_argument('--log', help='Location of log file', default=None)
# <add arguments here>
#args = parser.parse_args()
## Set up Logging
#if args.log is not None:
#    logging.basicConfig(filename=args.log, level=args.loglevel)
#else:
#    logging.basicConfig(level=args.loglevel)
# <add other setup here>
# create titan config
titan_config = TITAN_Config('http://{0}:{1}/graphs/{2}'.format(TITAN_HOST, TITAN_PORT, TITAN_GRAPH))
neo4j_config = NEO_Config(NEO4J_URI)


## EXECUTION
def removeNonAscii(s): return "".join(i for i in s if ord(i)<128)


#def merge_neo4j(g, neo4j=neo4j_config):  # Bulbs
def merge_neo4j(g, neo4j="http://localhost:7474/db/data/"):  # Neo4j
    """

    :param g: networkx graph to be merged
    :param neo4j: bulbs neo4j config
    :return: Nonetype

    Note: Neo4j operates differently from the current titan import.  The neo4j import does not aggregate edges which
           means they must be handled at query time.  The current titan algorithm aggregates edges based on time on
           merge.
    """
    #neo4j_graph = NEO_Graph(neo4j)  # Bulbs
    neo_graph = py2neoGraph(neo4j)
    nodes = set()
    edges = set()
    settled = set()
    for edge in g.edges(data=True):
#        print edge  # DEBUG
        # Get the src node
        src_uri = edge[0]
        if src_uri not in settled:
            attr = g.node[src_uri]
            # get/create the src node
            # src = neo4j_graph.vertices.get_or_create("uri", src_uri, attr)  # Bulbs
            src = neo_graph.merge_one(attr['class'], 'uri', src_uri)
            src.set_properties(attr)
            nodes.add(src)
            settled.add(src_uri)
            # TODO: set "start_time" and "finish_time" to dummie variables in attr.
            # TODO:  Add nodes to graph, and cyper/gremlin query to compare to node start_time & end_time to dummie
            # TODO:  variable update if node start > dummie start & node finish < dummie finish, and delete dummie
            # TODO:  variables.

        # get/create the dst node
        dst_uri = edge[1]
        if dst_uri not in settled:
            # dst = neo4j_graph.vertices.get_or_create("uri", src_uri, attr)  # Bulbs
            dst = neo_graph.merge_one(attr['class'], 'uri', src_uri)
            dst.set_properties(attr)
            nodes.add(dst)
            settled.add(dst_uri)
            # TODO: set "start_time" and "finish_time" to dummie variables in attr.
            # TODO:  Add nodes to graph, and cyper/gremlin query to compare to node start_time & end_time to dummie
            # TODO:  variable update if node start > dummie start & node finish < dummie finish, and delete dummie
            # TODO:  variables.

        # create the edge
        # NOTE: No attempt is made to deduplicate edges between the graph to be merged and the destination graph.
        #        The query scripts should handle this.
        try:
            relationship = edge[2].pop('relationship')
        except:
            # default to 'described_by'
            relationship = 'describedBy'
        rel = py2neoRelationship(src_uri, relationship, dst_uri)
        rel.set_properties(edge[2])
        edges.add(rel)

    # push updates to nodes all at once
    neo_graph.push(*nodes)
    # create edges all at once
    neo_graph.create(*edges)


def merge_titandb(g, titan=titan_config):
    """
<commit_msg>Tested neo4j merge function.  Batch update not working for nodes.  Instead nodes are individually pushed.<commit_after>#!/usr/bin/env python
"""
 AUTHOR: <NAME>
 DATE: 12-17-2013
 DEPENDENCIES: a list of modules requiring installation
 Copyright 2014 <NAME>

 LICENSE:
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.

 DESCRIPTION:
 Functions necessary to enrich the context graph

"""
# PRE-USER SETUP
from datetime import timedelta

########### NOT USER EDITABLE ABOVE THIS POINT #################


# USER VARIABLES
MAXMIND_FILE = "./GeoIPASNum.dat"
TITAN_HOST = "localhost"
TITAN_PORT = "8182"
TITAN_GRAPH = "vzgraph"
STATES = {'AA': 'armed forces americas', 'AE': 'armed forces middle east', 'AK': 'alaska', 'AL': 'alabama',
          'AP': 'armed forces pacific', 'AR': 'arkansas', 'AS': 'american samoa', 'AZ': 'arizona', 'CA': 'california',
          'CO': 'colorado', 'CT': 'connecticut', 'DC': 'district of columbia', 'DE': 'delaware', 'FL': 'florida',
          'FM': 'federated states of micronesia', 'GA': 'georgia', 'GU': 'guam', 'HI': 'hawaii', 'IA': 'iowa',
          'ID': 'idaho', 'IL': 'illinois', 'IN': 'indiana', 'KS': 'kansas', 'KY': 'kentucky', 'LA': 'louisiana',
          'MA': 'massachusetts', 'MD': 'maryland', 'ME': 'maine', 'MH': 'marshall islands', 'MI': 'michigan',
          'MN': 'minnesota', 'MO': 'missouri', 'MP': 'northern mariana islands', 'MS': 'mississippi', 'MT': 'montana',
          'NC': 'north carolina', 'ND': 'north dakota', 'NE': 'nebraska', 'NH': 'new hampshire', 'NJ': 'new jersey',
          'NM': 'new mexico', 'NV': 'nevada', 'NY': 'new york', 'OH': 'ohio', 'OK': 'oklahoma', 'OR': 'oregon',
          'PA': 'pennsylvania', 'PR': 'puerto rico', 'PW': 'palau', 'RI': 'rhode island', 'SC': 'south carolina',
          'SD': 'south dakota', 'TN': 'tennessee', 'TX': 'texas', 'UT': 'utah', 'VA': 'virginia',
          'VI': 'virgin islands', 'VT': 'vermont', 'WA': 'washington', 'WI': 'wisconsin', 'WV': 'west virginia',
          'WY': 'wyoming'}



########### NOT USER EDITABLE BELOW THIS POINT #################


## IMPORTS
import networkx as nx
import tldextract
import argparse
import logging
import sys
from datetime import datetime # timedelta imported above
import uuid
import copy
from cymru import CymruIPtoASNService
from cags_schema import TalksTo, DescribedBy, Influences
import GeoIP
from bulbs.titan import Graph as TITAN_Graph
from bulbs.titan import Config as TITAN_Config
#import csv
import ipaddress
import pandas as pd
from urlparse import urlparse
from collections import defaultdict
import socket
import tldextract
from bulbs.neo4jserver import NEO4J_URI
from bulbs.neo4jserver import Graph as NEO_Graph
from bulbs.neo4jserver import Config as NEO_Config
from py2neo import Graph as py2neoGraph
from py2neo import Relationship as py2neoRelationship
from py2neo import Node as py2neoNode

## SETUP
__author__ = "<NAME>"
# Parse Arguments (should correspond to user variables)
parser = argparse.ArgumentParser(description='This script processes a graph.')
parser.add_argument('-d', '--debug',
                    help='Print lots of debugging statements',
                    action="store_const", dest="loglevel", const=logging.DEBUG,
                    default=logging.WARNING
                   )
parser.add_argument('-v', '--verbose',
                    help='Be verbose',
                    action="store_const", dest="loglevel", const=logging.INFO
                   )
parser.add_argument('--log', help='Location of log file', default=None)
# <add arguments here>
#args = parser.parse_args()
## Set up Logging
#if args.log is not None:
#    logging.basicConfig(filename=args.log, level=args.loglevel)
#else:
#    logging.basicConfig(level=args.loglevel)
# <add other setup here>
# create titan config
titan_config = TITAN_Config('http://{0}:{1}/graphs/{2}'.format(TITAN_HOST, TITAN_PORT, TITAN_GRAPH))
neo4j_config = NEO_Config(NEO4J_URI)


## EXECUTION
def removeNonAscii(s): return "".join(i for i in s if ord(i)<128)


#def merge_neo4j(g, neo4j=neo4j_config):  # Bulbs
def merge_neo4j(g, neo4j="http://localhost:7474/db/data/"):  # Neo4j
    """

    :param g: networkx graph to be merged
    :param neo4j: bulbs neo4j config
    :return: Nonetype

    Note: Neo4j operates differently from the current titan import.  The neo4j import does not aggregate edges which
           means they must be handled at query time.  The current titan algorithm aggregates edges based on time on
           merge.
    """
    #neo4j_graph = NEO_Graph(neo4j)  # Bulbs
    neo_graph = py2neoGraph(neo4j)
    nodes = set()
    edges = set()
    settled = set()
    for edge in g.edges(data=True):
#        print edge  # DEBUG
        # Get the src node
        src_uri = edge[0]
        if src_uri not in settled:
            attr = g.node[src_uri]
            # get/create the src node
            # src = neo4j_graph.vertices.get_or_create("uri", src_uri, attr)  # Bulbs
            src = neo_graph.merge_one(attr['class'], 'uri', src_uri)
            print src  # DEBUG
            src.set_properties(attr)
            #src.cast(attr)  # Attempt to push all node updates at same time 1/3
            nodes.add(src)
            settled.add(src_uri)
            # TODO: set "start_time" and "finish_time" to dummie variables in attr.
            # TODO:  Add nodes to graph, and cyper/gremlin query to compare to node start_time & end_time to dummie
            # TODO:  variable update if node start > dummie start & node finish < dummie finish, and delete dummie
            # TODO:  variables.

        # get/create the dst node
        dst_uri = edge[1]
        if dst_uri not in settled:
            attr = g.node[dst_uri]
            # dst = neo4j_graph.vertices.get_or_create("uri", src_uri, attr)  # Bulbs
            dst = neo_graph.merge_one(attr['class'], 'uri', dst_uri)
            print dst  # DEBUG
            dst.set_properties(attr)
            #dst.cast(attr)  # Attempt to push all node updates at same time 2/3
            nodes.add(dst)
            settled.add(dst_uri)
            # TODO: set "start_time" and "finish_time" to dummie variables in attr.
            # TODO:  Add nodes to graph, and cyper/gremlin query to compare to node start_time & end_time to dummie
            # TODO:  variable update if node start > dummie start & node finish < dummie finish, and delete dummie
            # TODO:  variables.

        # create the edge
        # NOTE: No attempt is made to deduplicate edges between the graph to be merged and the destination graph.
        #        The query scripts should handle this.
        try:
            relationship = edge[2].pop('relationship')
        except:
            # default to 'described_by'
            relationship = 'describedBy'
        rel = py2neoRelationship(src, relationship, dst)
        rel.set_properties(edge[2])
        neo_graph.create(rel)  # Debug
        edges.add(rel)

#    return nodes, edges  # Debug
    # push updates to nodes all at once
    print nodes  # Debug
    #neo_graph.push(*nodes)  # Attempt to push all node updates at same time 3/3
    # create edges all at once
    print edges  # Debug
 #   neo_graph.create(*edges)


def merge_titandb(g, titan=titan_config):
    """

{'fraction_non_alphanumeric': 0.12032520325203253, 'fraction_numerical': 0.00865040650406504, 'mean_word_length': 3.16017316017316, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3054720', 'n_tokens_mistral': 4921, 'n_tokens_neox': 4683, 'n_words': 1614}