<commit_before>from data_interfaces import graph_database
import calendar
import re


class BaseDataModel:
    def __init__(self):
        self.g = graph_database.GraphInterface()
        self.vertex = None
        self.label = None
        self.document_label = 'Document'
        self.named_label = "Named Entity"

    def fetch(self, label, primary_attribute, search):
        exists = False
        self.vertex = self.find_vertex(
            label, primary_attribute, search
        )
        if self.vertex:
            exists = True
        return exists

    def find_vertex(self, label, node_key, value):
        search_query = u"""
                MATCH (v:`{0}` {{{1}:"{2}"}})
                RETURN v
            """.format(label, node_key, value)
        output = self.query(search_query)
        if output:
            return output[0][0]
        else:
            return None

    def create_vertex(self, label, node_key, value, merge=True):
        self.vertex = None
        if merge:
            search_query = u"""
                    MERGE (v:`{0}` {{{1}:"{2}"}})
                    ON MATCH set v:`{0}`
                    ON CREATE set v:`{0}`
                    RETURN v
                """.format(label, node_key, value)
        else:
            search_query = u"""
                    CREATE (v:`{0}` {{{1}:"{2}"}})
                    RETURN v
                """.format(label, node_key, value)
            #print search_query
        output = self.query(search_query)
        self.vertex = output[0][0]
        self.vertex.add_labels(label)
        return self.vertex

    def set_node_properties(self, properties=None, labels=None):
        if properties:
            node_properties = self.vertex.get_properties()
            for prop in properties:
                self.vertex.properties[prop] = properties[prop]
        if labels:
            if isinstance(labels, list):
                for label in labels:
                    self.vertex.labels.add(label)
            else:
                self.vertex.labels.add(labels)
        self.vertex.push()

    def create_relationship(self, vertex1, relationship, vertex2):
        rel_query = u"""
            START n=node({0}), m=node({1})
            MERGE (n)-[r:{2}]-(m)
            RETURN r
        """.format(vertex1._id, vertex2._id, relationship)
        return self.query(rel_query)

    def query(self, query_string):
        return self.g.graph.cypher.execute(query_string)

    def get_all_nodes(self, node_type):
        search_string = u"MATCH (n:`{0}`) RETURN n".format(node_type)
        output = self.query(search_string)
        for result in output:
            yield result[0]

    def set_date(self, date, relationship):
        if '/' in date:
            d = date.split('/')
            month, day, year = int(d[0]), int(d[1]), int(d[2])
        elif '-' in date:
            d = date.split('-')
            year, month, day = int(d[0]), int(d[1]), int(d[2])
        elif ' ' in date:
            d = date.split(' ')
            month_digit = self._convert_month(d[1])
            day, month, year = int(d[0]), int(month_digit), int(d[2])
        self.create_relationship(
            self.vertex,
            relationship,
            self.g.calendar.date(year, month, day).day
        )

    @staticmethod
    def _format_content(string):
        old_content = string.split("\n\n")
        new_content = ""
        for para in old_content:<commit_msg>added Named Entity dump /export<commit_after>from data_interfaces import graph_database
import calendar
import re


class BaseDataModel:
    def __init__(self):
        self.g = graph_database.GraphInterface()
        self.vertex = None
        self.label = None
        self.document_label = 'Document'
        self.named_label = "Named Entity"

    def fetch(self, label, primary_attribute, search):
        exists = False
        self.vertex = self.find_vertex(
            label, primary_attribute, search
        )
        if self.vertex:
            exists = True
        return exists

    def find_vertex(self, label, node_key, value):
        search_query = u"""
                MATCH (v:`{0}` {{{1}:"{2}"}})
                RETURN v
            """.format(label, node_key, value)
        output = self.query(search_query)
        if output:
            return output[0][0]
        else:
            return None

    def create_vertex(self, label, node_key, value, merge=True):
        self.vertex = None
        if merge:
            search_query = u"""
                    MERGE (v:`{0}` {{{1}:"{2}"}})
                    ON MATCH set v:`{0}`
                    ON CREATE set v:`{0}`
                    RETURN v
                """.format(label, node_key, value)
        else:
            search_query = u"""
                    CREATE (v:`{0}` {{{1}:"{2}"}})
                    RETURN v
                """.format(label, node_key, value)
            #print search_query
        output = self.query(search_query)
        self.vertex = output[0][0]
        self.vertex.add_labels(label)
        return self.vertex

    def set_node_properties(self, properties=None, labels=None):
        if properties:
            node_properties = self.vertex.get_properties()
            for prop in properties:
                self.vertex.properties[prop] = properties[prop]
        if labels:
            if isinstance(labels, list):
                for label in labels:
                    self.vertex.labels.add(label)
            else:
                self.vertex.labels.add(labels)
        self.vertex.push()

    def create_relationship(self, vertex1, relationship, vertex2):
        rel_query = u"""
            START n=node({0}), m=node({1})
            MERGE (n)-[r:{2}]-(m)
            RETURN r
        """.format(vertex1._id, vertex2._id, relationship)
        return self.query(rel_query)

    def query(self, query_string):
        return self.g.graph.cypher.execute(query_string)

    def get_all_nodes(self, node_type):
        search_string = u"MATCH (n:`{0}`) RETURN n".format(node_type)
        output = self.query(search_string)
        for result in output:
            yield result[0]

    def set_date(self, date, relationship):
        if '/' in date:
            d = date.split('/')
            month, day, year = int(d[0]), int(d[1]), int(d[2])
        elif '-' in date:
            d = date.split('-')
            year, month, day = int(d[0]), int(d[1]), int(d[2])
        elif ' ' in date:
            d = date.split(' ')
            month_digit = self._convert_month(d[1])
            day, month, year = int(d[0]), int(month_digit), int(d[2])
        self.create_relationship(
            self.vertex,
            relationship,
            self.g.calendar.date(year, month, day).day
        )

    def named_entity_export(self):
        search_query = u"""
                MATCH (n:`Named Entity`)
                RETURN n.name, labels(n)
            """
        output = self.query(search_query)
        if output:
            for result in output:
                try:
                    print result[0], result[1]
                except UnicodeEncodeError:
                    pass

    @staticmethod
    def _format_content(string):
        old_content = string.split("\n\n")
        new_content = ""
        for para in old_content:
{'fraction_non_alphanumeric': 0.12112522190359143, 'fraction_numerical': 0.009285811825754471, 'mean_word_length': 1.6956201693043798, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1634957', 'n_tokens_mistral': 2156, 'n_tokens_neox': 2073, 'n_words': 498}