<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import io
import time

from devstack import cfg
from devstack import component as comp
from devstack import log as logging
from devstack import settings
from devstack import shell as sh
from devstack import utils

from devstack.components import db

LOG = logging.getLogger("devstack.components.melange")

#id
TYPE = settings.MELANGE

#the pkg json files melange requires for installation
REQ_PKGS = ['general.json', 'melange.json']

#this db will be dropped then created
DB_NAME = 'melange'

#subdirs of the checkout/download
BIN_DIR = 'bin'

#configs
ROOT_CONF = 'melange.conf.sample'
ROOT_CONF_REAL_NAME = 'melange.conf'
CONFIGS = [ROOT_CONF]
CFG_LOC = ['etc', 'melange']

#how we sync melange with the db
DB_SYNC_CMD = [
    {'cmd': ['%BINDIR%/melange-manage', '--config-file', '%CFG_FILE%',
             'db_sync']},
]

#???
CIDR_CREATE_CMD = [
    {'cmd': ['melange', 'mac_address_range', 'create', 'cidr', '%CIDR_RANGE%']},
]

#what to start
APP_OPTIONS = {
    'melange-server': ['--config-file', '%CFG_FILE%'],
}

#subcomponent that specifies we should make the network cidr using melange
CREATE_CIDR = "create-cidr"
WAIT_ONLINE_TO = 5


class MelangeUninstaller(comp.PythonUninstallComponent):
    def __init__(self, *args, **kargs):
        comp.PythonUninstallComponent.__init__(self, TYPE, *args, **kargs)


class MelangeInstaller(comp.PythonInstallComponent):
    def __init__(self, *args, **kargs):
        comp.PythonInstallComponent.__init__(self, TYPE, *args, **kargs)
        self.bindir = sh.joinpths(self.appdir, BIN_DIR)
        self.cfgdir = sh.joinpths(self.appdir, *CFG_LOC)

    def _get_download_locations(self):
        places = list()
        places.append({
            'uri': ("git", "melange_repo"),
            'branch': ("git", "melange_branch"),
        })
        return places

    def _setup_db(self):
        LOG.info("Fixing up database named %s.", DB_NAME)
        db.drop_db(self.cfg, DB_NAME)
        db.create_db(self.cfg, DB_NAME)

    def _get_pkgs(self):
        return list(REQ_PKGS)

    def post_install(self):
        comp.PythonInstallComponent.post_install(self)
        self._setup_db()
        self._sync_db()

    def _sync_db(self):
        LOG.info("Syncing the database with melange.")
        mp = dict()
        mp['BINDIR'] = self.bindir
        mp['CFG_FILE'] = sh.joinpths(self.cfgdir, ROOT_CONF_REAL_NAME)
        utils.execute_template(*DB_SYNC_CMD, params=mp)

    def _get_config_files(self):
        return list(CONFIGS)

    def _config_adjust(self, contents, config_fn):
        if config_fn == ROOT_CONF:
            newcontents = contents
            with io.BytesIO(contents) as stream:
                config = cfg.IgnoreMissingConfigParser()
                config.readfp(stream)
                db_dsn = self.cfg.get_dbdsn(DB_NAME)
                config.set('DEFAULT', 'sql_connection', db_dsn)
                with io.BytesIO() as outputstream:
                    config.write(outputstream)
                    outputstream.flush()
                    new_data = ['# Adjusted %s' % (config_fn), outputstream.getvalue()]
                    #TODO can we write to contents here directly?
                    newcontents = utils.joinlinesep(*new_data)
            contents = newcontents
        return contents

    def _get_source_config(self, config_fn):
        if config_fn == ROOT_CONF:
            srcfn = sh.joinpths(self.cfgdir, config_fn)<commit_msg>Some cleanups, it still seems broke due to the melange code not working.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import io
import time

from devstack import cfg
from devstack import component as comp
from devstack import log as logging
from devstack import settings
from devstack import shell as sh
from devstack import utils

from devstack.components import db

LOG = logging.getLogger("devstack.components.melange")

#id
TYPE = settings.MELANGE

#the pkg json files melange requires for installation
REQ_PKGS = ['general.json', 'melange.json']

#this db will be dropped then created
DB_NAME = 'melange'

#subdirs of the checkout/download
BIN_DIR = 'bin'

#configs
ROOT_CONF = 'melange.conf.sample'
ROOT_CONF_REAL_NAME = 'melange.conf'
CONFIGS = [ROOT_CONF]
CFG_LOC = ['etc', 'melange']

#how we sync melange with the db
DB_SYNC_CMD = [
    {'cmd': ['%BINDIR%/melange-manage', '--config-file=%CFG_FILE%', 'db_sync']},
]

#???
CIDR_CREATE_CMD = [
    {'cmd': ['melange', 'mac_address_range', 'create', 'cidr', '%CIDR_RANGE%']},
]

#what to start
APP_OPTIONS = {
    'melange-server': ['--config-file', '%CFG_FILE%'],
}

#subcomponent that specifies we should make the network cidr using melange
CREATE_CIDR = "create-cidr"
WAIT_ONLINE_TO = 5


class MelangeUninstaller(comp.PythonUninstallComponent):
    def __init__(self, *args, **kargs):
        comp.PythonUninstallComponent.__init__(self, TYPE, *args, **kargs)


class MelangeInstaller(comp.PythonInstallComponent):
    def __init__(self, *args, **kargs):
        comp.PythonInstallComponent.__init__(self, TYPE, *args, **kargs)
        self.bindir = sh.joinpths(self.appdir, BIN_DIR)
        self.cfgdir = sh.joinpths(self.appdir, *CFG_LOC)

    def _get_download_locations(self):
        places = list()
        places.append({
            'uri': ("git", "melange_repo"),
            'branch': ("git", "melange_branch"),
        })
        return places

    def _setup_db(self):
        LOG.info("Fixing up database named %s.", DB_NAME)
        db.drop_db(self.cfg, DB_NAME)
        db.create_db(self.cfg, DB_NAME)

    def _get_pkgs(self):
        return list(REQ_PKGS)

    def post_install(self):
        comp.PythonInstallComponent.post_install(self)
        self._setup_db()
        self._sync_db()

    def _sync_db(self):
        LOG.info("Syncing the database with melange.")
        mp = dict()
        mp['BINDIR'] = self.bindir
        mp['CFG_FILE'] = sh.joinpths(self.cfgdir, ROOT_CONF_REAL_NAME)
        utils.execute_template(*DB_SYNC_CMD, params=mp)

    def _get_config_files(self):
        return list(CONFIGS)

    def _config_adjust(self, contents, config_fn):
        if config_fn == ROOT_CONF:
            newcontents = contents
            with io.BytesIO(contents) as stream:
                config = cfg.IgnoreMissingConfigParser()
                config.readfp(stream)
                db_dsn = self.cfg.get_dbdsn(DB_NAME)
                old_dbsn = config.get('DEFAULT', 'sql_connection')
                if db_dsn != old_dbsn:
                    config.set('DEFAULT', 'sql_connection', db_dsn)
                    with io.BytesIO() as outputstream:
                        config.write(outputstream)
                        outputstream.flush()
                        new_data = ['# Adjusted %s' % (config_fn), outputstream.getvalue()]
                        #TODO can we write to contents here directly?
                        newcontents = utils.joinlinesep(*new_data)
            contents = newcontents
        return contents

    def _get_source_config(self, config_fn):
        if config_fn == ROOT_CONF:
            srcfn = sh.joinpths(self.cfgdir, config_fn)
{'fraction_non_alphanumeric': 0.10624925886398671, 'fraction_numerical': 0.002845962290999644, 'mean_word_length': 2.9064381658175082, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7295199', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2578, 'n_words': 710}