<commit_before># -*- coding: utf-8 -*-
"""Setup the pyforge application"""
import os
import sys
import logging
import shutil
from datetime import datetime
from mimetypes import guess_type
from tg import config
from pylons import c, g
from paste.deploy.converters import asbool

from flyway.model import MigrationInfo
from ming import Session
from ming.orm.ormsession import ThreadLocalORMSession

import pyforge
from pyforge import model as M

log = logging.getLogger(__name__)

def set_flyway_info():
    versions = dict(
        pyforge=1,
        ForgeTracker=0,
        ForgeWiki=0)
    mi = MigrationInfo.make(dict(versions=versions))
    conn = M.main_doc_session.bind.conn
    for database in conn.database_names():
        log.info('Initialize Flyway for %s', database)
        session = DBSession(conn[database])
        session.insert(mi)

def cache_test_data():
    log.info('Saving data to cache in .test-data')
    if os.path.exists('.test-data'):
        shutil.rmtree('.test-data')
    os.system('mongodump -h 127.0.0.1:27018 -o .test-data > mongodump.log 2>&1')

def restore_test_data():
    if os.path.exists('.test-data'):
        log.info('Restoring data from cache in .test-data')
        rc = os.system('mongorestore -h 127.0.0.1:27018 --dir .test-data > mongorestore.log 2>&1')
        return rc == 0
    else:
        return False

def bootstrap(command, conf, vars):
    """Place any commands to setup pyforge here"""
    # Clean up all old stuff
    ThreadLocalORMSession.close_all()
    c.queued_messages = []
    database=conf.get('db_prefix', '') + 'project:test'
    wipe_database()
    set_flyway_info()
    g._push_object(pyforge.lib.app_globals.Globals())
    try:
        g.solr.delete(q='*:*')
    except: # pragma no cover
        log.error('SOLR server is %s', g.solr_server)
        log.error('Error clearing solr index')
    if asbool(conf.get('cache_test_data')):
        if restore_test_data():
            c.project = M.Project.query.get(shortname='test')
            return
    log.info('Initializing search')
    M.SearchConfig(last_commit = datetime.utcnow(),
                   pending_commit = 0)
    g.publish('audit', 'search.check_commit', {})
    log.info('Registering initial users & neighborhoods')
    anonymous = M.User(_id=None,
                       username='*anonymous',
                       display_name='<NAME>')
    root = create_user('Root')
    n_projects = create_neighborhood('Projects', root)
    n_users = M.Neighborhood(name='Users',
                             url_prefix='/users/',
                             shortname_prefix='users/',
                             acl=dict(read=[None], create=[],
                                      moderate=[root._id], admin=[root._id]))
    n_adobe = M.Neighborhood(name='Adobe',
                             url_prefix='/adobe/',
                             acl=dict(read=[None], create=[],
                                      moderate=[root._id], admin=[root._id]))
    M.Theme(name='forge_default',
            label='Default Forge Theme',
            color1='#0088cc',
            color2='#000000',
            color3='#454545',
            color4='#6c7681',
            color5='#d8d8d8',
            color6='#ececec')
    ThreadLocalORMSession.flush_all()
    ThreadLocalORMSession.close_all()
    # add the adobe icon
    file_name = 'adobe_icon.png'
    file_path = os.path.join(pyforge.__path__[0],'public','images',file_name)
    f = file(file_path, 'r')
    content_type = guess_type(file_name)
    if content_type: content_type = content_type[0]
    else: content_type = 'application/octet-stream'
    with M.NeighborhoodFile.create(
        content_type=content_type,
        filename=file_name,
        neighborhood_id=n_adobe._id) as fp:
        while True:
            s = f.read()
            if not s: break
            fp.write(s)
    log.info('Registering "regular users" (non-root)')
    u_adobe = M.User.register(dict(username='adobe_admin',
                                   display_name='Adobe Admin'))
    u0 = M.User.register(dict(username='test_admin',
                              display_name='Test Admin'))
    u1 = M.User.register(dict(username='test_user',
                              display_name='Test User'))
    u2 = M.User.register(dict(username='test_user2',
                              display_name='Test User 2'))
    n_adobe.acl['admin'].append(u_adobe._id)
    u_adobe.set_password('<PASSWORD>')
    u0.set_password('<PASSWORD>')
    u1.set_password('<PASSWORD>')
    u0.claim_address('<EMAIL>')

    log.info('Creating basic project categories')
    cat1 = M.ProjectCategory(name='clustering', label='Clustering')

    cat2 = M.ProjectCategory(name='communications', label='Communications')
    cat2_1 = M.ProjectCategory(name='synchronization', label='Synchronization', parent_id=cat2._id)
    cat2_2 = M.ProjectCategory(name='streaming', label='Streaming', parent_id=cat2._id)
    cat2_3 = M.ProjectCategory(name='fax', label='Fax', parent_id=cat2._id)
    cat2_4 = M.ProjectCategory(name='bbs', label='BBS', parent_id=cat2._id)

    cat3 = M.ProjectCategory(name='database', label='Database')
    cat3_1 = M.ProjectCategory(name='front_ends', label='Front-Ends', parent_id=cat3._id)
    cat3_2 = M.ProjectCategory(name='engines_servers', label='Engines/Servers', parent_id=cat3._id)

    log.info('Registering initial projects')
    p_adobe1 = n_adobe.register_project('adobe_1', u_adobe)
    p_adobe2 = n_adobe.register_project('adobe_2', u_adobe)
    p0 = n_projects.register_project('test', u0)
    c.project = p0
    c.user = u0
    p1 = p0.new_subproject('sub1')
    ThreadLocalORMSession.flush_all()
    if asbool(conf.get('load_test_data')):
        log.info('Loading test data')
        app = p0.install_app('hello_forge', 'hello')
        app = p0.install_app('Repository', 'src')
        app = p0.install_app('Repository', 'src_git')
        p0.install_app('Wiki', 'wiki')
        p0.install_app('Tickets', 'bugs')
        p0.install_app('Tickets', 'doc_bugs')
        p0.install_app('Discussion', 'discussion')
        app.config.options['type'] = 'git'
        ThreadLocalORMSession.flush_all()
        ThreadLocalORMSession.close_all()
        if asbool(conf.get('cache_test_data')):
            cache_test_data()
    else: # pragma no cover
        log.info('Loading some large data')
        p0.install_app('Wiki', 'wiki')
        p0.install_app('Tickets', 'bugs')
        p0.install_app('Discussion', 'discussion')
        # app = p0.install_app('Repository', 'src')
        # app = p0.install_app('Repository', 'src_git')
        ThreadLocalORMSession.flush_all()
        for msg in c.queued_messages:<commit_msg>ForgeSCM is gone, so use the new tools in bootstrap.py<commit_after># -*- coding: utf-8 -*-
"""Setup the pyforge application"""
import os
import sys
import logging
import shutil
from datetime import datetime
from mimetypes import guess_type
from tg import config
from pylons import c, g
from paste.deploy.converters import asbool

from flyway.model import MigrationInfo
from ming import Session
from ming.orm.ormsession import ThreadLocalORMSession

import pyforge
from pyforge import model as M

log = logging.getLogger(__name__)

def set_flyway_info():
    versions = dict(
        pyforge=1,
        ForgeTracker=0,
        ForgeWiki=0)
    mi = MigrationInfo.make(dict(versions=versions))
    conn = M.main_doc_session.bind.conn
    for database in conn.database_names():
        log.info('Initialize Flyway for %s', database)
        session = DBSession(conn[database])
        session.insert(mi)

def cache_test_data():
    log.info('Saving data to cache in .test-data')
    if os.path.exists('.test-data'):
        shutil.rmtree('.test-data')
    os.system('mongodump -h 127.0.0.1:27018 -o .test-data > mongodump.log 2>&1')

def restore_test_data():
    if os.path.exists('.test-data'):
        log.info('Restoring data from cache in .test-data')
        rc = os.system('mongorestore -h 127.0.0.1:27018 --dir .test-data > mongorestore.log 2>&1')
        return rc == 0
    else:
        return False

def bootstrap(command, conf, vars):
    """Place any commands to setup pyforge here"""
    # Clean up all old stuff
    ThreadLocalORMSession.close_all()
    c.queued_messages = []
    database=conf.get('db_prefix', '') + 'project:test'
    wipe_database()
    set_flyway_info()
    g._push_object(pyforge.lib.app_globals.Globals())
    try:
        g.solr.delete(q='*:*')
    except: # pragma no cover
        log.error('SOLR server is %s', g.solr_server)
        log.error('Error clearing solr index')
    if asbool(conf.get('cache_test_data')):
        if restore_test_data():
            c.project = M.Project.query.get(shortname='test')
            return
    log.info('Initializing search')
    M.SearchConfig(last_commit = datetime.utcnow(),
                   pending_commit = 0)
    g.publish('audit', 'search.check_commit', {})
    log.info('Registering initial users & neighborhoods')
    anonymous = M.User(_id=None,
                       username='*anonymous',
                       display_name='<NAME>')
    root = create_user('Root')
    n_projects = create_neighborhood('Projects', root)
    n_users = M.Neighborhood(name='Users',
                             url_prefix='/users/',
                             shortname_prefix='users/',
                             acl=dict(read=[None], create=[],
                                      moderate=[root._id], admin=[root._id]))
    n_adobe = M.Neighborhood(name='Adobe',
                             url_prefix='/adobe/',
                             acl=dict(read=[None], create=[],
                                      moderate=[root._id], admin=[root._id]))
    M.Theme(name='forge_default',
            label='Default Forge Theme',
            color1='#0088cc',
            color2='#000000',
            color3='#454545',
            color4='#6c7681',
            color5='#d8d8d8',
            color6='#ececec')
    ThreadLocalORMSession.flush_all()
    ThreadLocalORMSession.close_all()
    # add the adobe icon
    file_name = 'adobe_icon.png'
    file_path = os.path.join(pyforge.__path__[0],'public','images',file_name)
    f = file(file_path, 'r')
    content_type = guess_type(file_name)
    if content_type: content_type = content_type[0]
    else: content_type = 'application/octet-stream'
    with M.NeighborhoodFile.create(
        content_type=content_type,
        filename=file_name,
        neighborhood_id=n_adobe._id) as fp:
        while True:
            s = f.read()
            if not s: break
            fp.write(s)
    log.info('Registering "regular users" (non-root)')
    u_adobe = M.User.register(dict(username='adobe_admin',
                                   display_name='Adobe Admin'))
    u0 = M.User.register(dict(username='test_admin',
                              display_name='Test Admin'))
    u1 = M.User.register(dict(username='test_user',
                              display_name='Test User'))
    u2 = M.User.register(dict(username='test_user2',
                              display_name='Test User 2'))
    n_adobe.acl['admin'].append(u_adobe._id)
    u_adobe.set_password('<PASSWORD>')
    u0.set_password('<PASSWORD>')
    u1.set_password('<PASSWORD>')
    u0.claim_address('<EMAIL>')

    log.info('Creating basic project categories')
    cat1 = M.ProjectCategory(name='clustering', label='Clustering')

    cat2 = M.ProjectCategory(name='communications', label='Communications')
    cat2_1 = M.ProjectCategory(name='synchronization', label='Synchronization', parent_id=cat2._id)
    cat2_2 = M.ProjectCategory(name='streaming', label='Streaming', parent_id=cat2._id)
    cat2_3 = M.ProjectCategory(name='fax', label='Fax', parent_id=cat2._id)
    cat2_4 = M.ProjectCategory(name='bbs', label='BBS', parent_id=cat2._id)

    cat3 = M.ProjectCategory(name='database', label='Database')
    cat3_1 = M.ProjectCategory(name='front_ends', label='Front-Ends', parent_id=cat3._id)
    cat3_2 = M.ProjectCategory(name='engines_servers', label='Engines/Servers', parent_id=cat3._id)

    log.info('Registering initial projects')
    p_adobe1 = n_adobe.register_project('adobe_1', u_adobe)
    p_adobe2 = n_adobe.register_project('adobe_2', u_adobe)
    p0 = n_projects.register_project('test', u0)
    c.project = p0
    c.user = u0
    p1 = p0.new_subproject('sub1')
    ThreadLocalORMSession.flush_all()
    if asbool(conf.get('load_test_data')):
        log.info('Loading test data')
        app = p0.install_app('hello_forge', 'hello')
        app = p0.install_app('SVN', 'src')
        app = p0.install_app('Git', 'src_git')
        p0.install_app('Wiki', 'wiki')
        p0.install_app('Tickets', 'bugs')
        p0.install_app('Tickets', 'doc_bugs')
        p0.install_app('Discussion', 'discussion')
        app.config.options['type'] = 'git'
        ThreadLocalORMSession.flush_all()
        ThreadLocalORMSession.close_all()
        if asbool(conf.get('cache_test_data')):
            cache_test_data()
    else: # pragma no cover
        log.info('Loading some large data')
        p0.install_app('Wiki', 'wiki')
        p0.install_app('Tickets', 'bugs')
        p0.install_app('Discussion', 'discussion')
        # app = p0.install_app('SVN', 'src')
        # app = p0.install_app('Git', 'src_git')
        ThreadLocalORMSession.flush_all()
        for msg in c.queued_messages:
{'fraction_non_alphanumeric': 0.134367310416822, 'fraction_numerical': 0.01729923197375289, 'mean_word_length': 2.7390577083914134, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5815421', 'n_tokens_mistral': 4393, 'n_tokens_neox': 4247, 'n_words': 784}