<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Appier Framework
# Copyright (c) 2008-2016 Hive Solutions Lda.
#
# This file is part of Hive Appier Framework.
#
# Hive Appier Framework is free software: you can redistribute it and/or modify
# it under the terms of the Apache License as published by the Apache
# Foundation, either version 2.0 of the License, or (at your option) any
# later version.
#
# Hive Appier Framework is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License for more details.
#
# You should have received a copy of the Apache License along with
# Hive Appier Framework. If not, see <http://www.apache.org/licenses/>.

__author__ = "<NAME> <<EMAIL>>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2016 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "Apache License, Version 2.0"
""" The license for the module """

import os
import json
import zipfile
import tempfile

from . import legacy

try: import bson
except: bson = None

IGNORE = 1
""" Ignore strategy for conflict solving in the import operation
basically this strategy skips importing a document that has the same
key value as one that already exists in the collection """

OVERWRITE = 2
""" Strategy for conflict solving that overwrites (completely) a
previously existing document in the data source if it has the same
key value as the one being imported, this should be used carefully
as it may create data loss """

DUPLICATE = 3
""" Conflict solving strategy that basically duplicates the entries
in the data source even if they have the same key value, this may
create a somehow inconsistent state and so must be used carefully """

JOIN = 4
""" Join strategy for conflict solving in document collision, that
basically adds new fields or updates existing fields in a previously
existing document, this strategy does not remove extra fields existing
in the previous document """

class ExportManager(object):

    db = None
    single = None
    multiple = None

    def __init__(self, db, single = (), multiple = ()):
        self.db = db
        self.single = single
        self.multiple = multiple

    def import_data(self, file_path, policy = IGNORE):
        temporary_path = tempfile.mkdtemp()
        base_path = temporary_path
        single_path = os.path.join(base_path, "settings")

        self._deploy_zip(file_path, temporary_path)

        for name, key in self.single:
            collection = self.db[name]
            source_path = os.path.join(single_path, "%s.json" % name)
            file = open(source_path, "rb")
            try: data = file.read()
            finally: file.close()
            self._import_single(
                collection,
                data,
                key = key,
                policy = policy
            )

        for name, key in self.multiple:
            source_directory = os.path.join(base_path, name)
            if not os.path.exists(source_directory): continue

            collection = self.db[name]
            items = os.listdir(source_directory)
            data = []

            for item in items:
                value, _extension = os.path.splitext(item)
                source_path = os.path.join(source_directory, item)
                file = open(source_path, "rb")
                try: _data = file.read()
                finally: file.close()

                data.append((value, _data))

            self._import_multiple(
                collection,
                data,
                key = key,
                policy = policy
            )

    def export_data(self, file_path):
        temporary_path = tempfile.mkdtemp()
        base_path = temporary_path
        single_path = os.path.join(base_path, "settings")
        if not os.path.exists(single_path): os.makedirs(single_path)

        for name, key in self.single:
            collection = self.db[name]
            data = self._export_single(collection, key)
            target_path = os.path.join(single_path, "%s.json" % name)
            file = open(target_path, "wb")
            try: file.write(data)
            finally: file.close()

        for name, key in self.multiple:
            collection = self.db[name]
            data = self._export_multiple(collection, key)

            target_directory = os.path.join(base_path, name)
            if not os.path.exists(target_directory): os.makedirs(target_directory)

            for value, _data in data:
                target_path = os.path.join(target_directory, "%s.json" % value)
                file = open(target_path, "wb")<commit_msg>new adapter export support<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Appier Framework
# Copyright (c) 2008-2016 Hive Solutions Lda.
#
# This file is part of Hive Appier Framework.
#
# Hive Appier Framework is free software: you can redistribute it and/or modify
# it under the terms of the Apache License as published by the Apache
# Foundation, either version 2.0 of the License, or (at your option) any
# later version.
#
# Hive Appier Framework is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License for more details.
#
# You should have received a copy of the Apache License along with
# Hive Appier Framework. If not, see <http://www.apache.org/licenses/>.

__author__ = "<NAME> <<EMAIL>>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2016 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "Apache License, Version 2.0"
""" The license for the module """

import os
import json
import zipfile
import tempfile

from . import legacy

try: import bson
except: bson = None

IGNORE = 1
""" Ignore strategy for conflict solving in the import operation
basically this strategy skips importing a document that has the same
key value as one that already exists in the collection """

OVERWRITE = 2
""" Strategy for conflict solving that overwrites (completely) a
previously existing document in the data source if it has the same
key value as the one being imported, this should be used carefully
as it may create data loss """

DUPLICATE = 3
""" Conflict solving strategy that basically duplicates the entries
in the data source even if they have the same key value, this may
create a somehow inconsistent state and so must be used carefully """

JOIN = 4
""" Join strategy for conflict solving in document collision, that
basically adds new fields or updates existing fields in a previously
existing document, this strategy does not remove extra fields existing
in the previous document """

class ExportManager(object):

    adapter = None
    single = None
    multiple = None

    def __init__(self, adapter, single = (), multiple = ()):
        self.adapter = adapter
        self.single = single
        self.multiple = multiple

    def import_data(self, file_path, policy = IGNORE):
        temporary_path = tempfile.mkdtemp()
        base_path = temporary_path
        single_path = os.path.join(base_path, "settings")

        self._deploy_zip(file_path, temporary_path)

        for name, key in self.single:
            collection = self.adapter.collection(name)
            source_path = os.path.join(single_path, "%s.json" % name)
            file = open(source_path, "rb")
            try: data = file.read()
            finally: file.close()
            self._import_single(
                collection,
                data,
                key = key,
                policy = policy
            )

        for name, key in self.multiple:
            source_directory = os.path.join(base_path, name)
            if not os.path.exists(source_directory): continue

            collection = self.collection(name)
            items = os.listdir(source_directory)
            data = []

            for item in items:
                value, _extension = os.path.splitext(item)
                source_path = os.path.join(source_directory, item)
                file = open(source_path, "rb")
                try: _data = file.read()
                finally: file.close()

                data.append((value, _data))

            self._import_multiple(
                collection,
                data,
                key = key,
                policy = policy
            )

    def export_data(self, file_path):
        temporary_path = tempfile.mkdtemp()
        base_path = temporary_path
        single_path = os.path.join(base_path, "settings")
        if not os.path.exists(single_path): os.makedirs(single_path)

        for name, key in self.single:
            collection = self.adapter.collection(name)
            data = self._export_single(collection, key)
            target_path = os.path.join(single_path, "%s.json" % name)
            file = open(target_path, "wb")
            try: file.write(data)
            finally: file.close()

        for name, key in self.multiple:
            collection = self.adapter.collection(name)
            data = self._export_multiple(collection, key)

            target_directory = os.path.join(base_path, name)
            if not os.path.exists(target_directory): os.makedirs(target_directory)

            for value, _data in data:
                target_path = os.path.join(target_directory, "%s.json" % value)
                file = open(target_path, "wb")
{'fraction_non_alphanumeric': 0.08762114097312419, 'fraction_numerical': 0.005594964531921271, 'mean_word_length': 2.5110487548228693, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1264815', 'n_tokens_mistral': 2776, 'n_tokens_neox': 2639, 'n_words': 994}