<commit_before># Copyright (c) 2016 Universidade Federal Fluminense (UFF)
# Copyright (c) 2016 Polytechnic Institute of New York University.
# This file is part of noWorkflow.
# Please, consult the license terms in the LICENSE file.
"""Content Database"""
from __future__ import (absolute_import, print_function,
                        division, unicode_literals)
import sys

from os.path import join, isdir, isfile
from ..persistence.content_database_engine import DistributedPyGitContentDatabaseEngine, \
    StandardContentDatabaseEngine, PyGitContentDatabaseEngine
from ..utils.io import print_msg

STANDARD_DATABASE_DIR = 'content'
GIT_DATABASE_DIR = 'content.git'


class ContentDatabase(object):
    """Content Database deal with storage of file content in disk"""

    def __init__(self, persistence_config):
        self.content_path = None  # Base path for storing content of files
        persistence_config.add(self)
        self.content_database_engine = None

    def set_path(self, config):
        """Set content_path"""

        config.content_dir = GIT_DATABASE_DIR
        self.content_path = join(config.provenance_path, config.content_dir)

        '''if isdir(join(config.provenance_path, STANDARD_DATABASE_DIR)):
            """Standard content database found"""
            config.content_dir = STANDARD_DATABASE_DIR
            self.content_path = join(config.provenance_path, config.content_dir)
        else:
            """if not use git content database"""
            config.content_dir = GIT_DATABASE_DIR
            self.content_path = join(config.provenance_path, config.content_dir)'''

    def connect(self, config):
        if config.content_dir == GIT_DATABASE_DIR:
            self.content_database_engine = DistributedPyGitContentDatabaseEngine(self.content_path)
            # self.content_database_engine = PyGitContentDatabaseEngine(self.content_path)
        else:
            self.content_database_engine = StandardContentDatabaseEngine(self.content_path)

        self.content_database_engine.connect()
<commit_msg>Fixing content database engine selection<commit_after># Copyright (c) 2016 Universidade Federal Fluminense (UFF)
# Copyright (c) 2016 Polytechnic Institute of New York University.
# This file is part of noWorkflow.
# Please, consult the license terms in the LICENSE file.
"""Content Database"""
from __future__ import (absolute_import, print_function,
                        division, unicode_literals)
import sys

from os.path import join, isdir, isfile
from ..persistence.content_database_engine import DistributedPyGitContentDatabaseEngine, \
    StandardContentDatabaseEngine, PyGitContentDatabaseEngine
from ..utils.io import print_msg

STANDARD_DATABASE_DIR = 'content'
GIT_DATABASE_DIR = 'content.git'


class ContentDatabase(object):
    """Content Database deal with storage of file content in disk"""

    def __init__(self, persistence_config):
        self.content_path = None  # Base path for storing content of files
        persistence_config.add(self)
        self.content_database_engine = None

    def set_path(self, config):
        """Set content_path"""

        if isdir(join(config.provenance_path, STANDARD_DATABASE_DIR)):
            """Standard content database found"""
            config.content_dir = STANDARD_DATABASE_DIR
            self.content_path = join(config.provenance_path, config.content_dir)
        else:
            """if not use git content database"""
            config.content_dir = GIT_DATABASE_DIR
            self.content_path = join(config.provenance_path, config.content_dir)

    def connect(self, config):
        if config.content_dir == GIT_DATABASE_DIR:
            self.content_database_engine = DistributedPyGitContentDatabaseEngine(self.content_path)
            # self.content_database_engine = PyGitContentDatabaseEngine(self.content_path)
        else:
            self.content_database_engine = StandardContentDatabaseEngine(self.content_path)

        self.content_database_engine.connect()

{'fraction_non_alphanumeric': 0.07588952475740234, 'fraction_numerical': 0.003981089823339139, 'mean_word_length': 3.6260069044879173, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1212825', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1075, 'n_words': 275}