<commit_before>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import os
import sys
import json
import shlex
import logging
import subprocess

from flask import Flask
from shutil import move

from manager_rest.storage.sql_models import db
from manager_rest.storage.storage_manager import get_storage_manager

format_str = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=format_str)
logger = logging.getLogger('estopg')

COMPUTE_NODE_TYPE = 'cloudify.nodes.Compute'
TYPES_TO_RESTORE = ['blueprint',
                    'deployment',
                    'execution',
                    'node',
                    'node_instance',
                    'plugin']


class EsToPg(object):
    def __init__(self, es_data_path):
        self._storage_manager = self._setup_server()
        self._blueprints_path = '{0}.blueprints'.format(es_data_path)
        self._deployments_path = '{0}.deployments'.format(es_data_path)
        self._deployments_ex_path = '{0}.deployments_ex'.format(es_data_path)
        self._nodes_path = '{0}.nodes'.format(es_data_path)
        self._node_instances_path = '{0}.node_instances'.format(es_data_path)
        self._plugins_path = '{0}.plugins'.format(es_data_path)
        self._executions_path = '{0}.executions'.format(es_data_path)
        self._events_path = '{0}.events'.format(es_data_path)

    @staticmethod
    def _setup_server():
        logger.debug('Connecting to PG db..')
        app = Flask(__name__)
        db_uri = 'postgresql://cloudify:cloudify@localhost/cloudify_db'
        app.config['SQLALCHEMY_DATABASE_URI'] = db_uri
        app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
        db.init_app(app)
        app.app_context().push()
        return get_storage_manager()

    def restore_es(self, es_data_path):
        logger.debug('Restoring elastic search..')
        self._split_dump(es_data_path)

        for line in open(self._blueprints_path, 'r'):
            elem = json.loads(line)
            node_data = elem['_source']
            self._update_blueprint(node_data)
            self._storage_manager.put_blueprint(node_data)

        for line in open(self._deployments_path, 'r'):
            elem = json.loads(line)
            node_data = elem['_source']
            self._update_deployment(node_data)
            self._storage_manager.put_deployment(node_data)

        for line in open(self._deployments_ex_path, 'r'):
            elem = json.loads(line)
            node_data = elem['_source']
            self._update_deployment(node_data)
            self._storage_manager.put_deployment(node_data)

        for line in open(self._nodes_path, 'r'):
            elem = json.loads(line)
            node_data = elem['_source']
            self._update_node(node_data)
            self._storage_manager.put_node(node_data)

        for line in open(self._node_instances_path, 'r'):
            elem = json.loads(line)
            node_data = elem['_source']
            self._update_node_instance(node_data)
            self._storage_manager.put_node_instance(node_data)<commit_msg>CFY-5958 handle old snapshot with deployment update<commit_after>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import os
import sys
import json
import shlex
import logging
import subprocess

from flask import Flask
from shutil import move

from manager_rest.storage.sql_models import db
from manager_rest.storage.storage_manager import get_storage_manager

format_str = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=format_str)
logger = logging.getLogger('estopg')

COMPUTE_NODE_TYPE = 'cloudify.nodes.Compute'
TYPES_TO_RESTORE = ['blueprint',
                    'deployment',
                    'execution',
                    'node',
                    'node_instance',
                    'plugin']


class EsToPg(object):
    def __init__(self, es_data_path):
        self._storage_manager = self._setup_server()
        self._blueprints_path = '{0}.blueprints'.format(es_data_path)
        self._deployments_path = '{0}.deployments'.format(es_data_path)
        self._deployments_ex_path = '{0}.deployments_ex'.format(es_data_path)
        self._nodes_path = '{0}.nodes'.format(es_data_path)
        self._node_instances_path = '{0}.node_instances'.format(es_data_path)
        self._plugins_path = '{0}.plugins'.format(es_data_path)
        self._executions_path = '{0}.executions'.format(es_data_path)
        self._events_path = '{0}.events'.format(es_data_path)

    @staticmethod
    def _setup_server():
        logger.debug('Connecting to PG db..')
        app = Flask(__name__)
        db_uri = 'postgresql://cloudify:cloudify@localhost/cloudify_db'
        app.config['SQLALCHEMY_DATABASE_URI'] = db_uri
        app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
        db.init_app(app)
        app.app_context().push()
        return get_storage_manager()

    def restore_es(self, es_data_path):
        logger.debug('Restoring elastic search..')
        self._split_dump(es_data_path)

        for line in open(self._blueprints_path, 'r'):
            elem = json.loads(line)
            node_data = elem['_source']
            self._update_blueprint(node_data)
            self._storage_manager.put_blueprint(node_data)

        for line in open(self._deployments_path, 'r'):
            elem = json.loads(line)
            node_data = elem['_source']
            self._update_deployment(node_data)
            self._storage_manager.put_deployment(node_data)

        for line in open(self._deployments_ex_path, 'r'):
            elem = json.loads(line)
            node_data = elem['_source']
            node_type = elem['_type']
            self._update_deployment(node_data)
            if node_type == 'deployment_updates':
                self._storage_manager.put_deployment_updates(node_data)
                if 'steps' in node_data:
                    steps = node_data['steps']
                    for step in steps:
                        self._storage_manager.put_deployment_update_step(step)
            elif node_type == 'deployment_modifications':
                self._storage_manager.put_deployment_modification(node_data)
            else:
                logger.warning('Unknown node type: {0}'.format(node_type))

        for line in open(self._nodes_path, 'r'):
            elem = json.loads(line)
            node_data = elem['_source']
            self._update_node(node_data)
            self._storage_manager.put_node(node_data)

        for line in open(self._node_instances_path, 'r'):
            elem = json.loads(line)
            node_data = elem['_source']
            self._update_node_instance(node_data)
            self._storage_manager.put_node_instance(node_data)
{'fraction_non_alphanumeric': 0.101295788686768, 'fraction_numerical': 0.004610017443309245, 'mean_word_length': 2.816928197812649, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4033910', 'n_tokens_mistral': 2401, 'n_tokens_neox': 2342, 'n_words': 508}