<commit_before>#!/usr/bin/env python

import argparse
from datetime import datetime
import imp
import os
import re
import shutil
import subprocess
import sys
import time
import warnings

from ConfigParser import SafeConfigParser
from loom.client.settings_manager import SettingsManager
from loom.client.common import *
from loom.common.version import version
import loom.common.cloud

DAEMON_EXECUTABLE = os.path.abspath(
    os.path.join(
    os.path.dirname(__file__),
    '../master/loomdaemon/loom_daemon.py'
    ))

GCLOUD_SERVER_DEFAULT_NAME = SettingsManager().get_default_setting('gcloud', 'SERVER_NAME')

PLAYBOOKS_PATH = os.path.join(imp.find_module('loom')[1], 'playbooks')
GCLOUD_CREATE_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_create_server.yml')
GCLOUD_START_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_start_server.yml')
GCLOUD_STOP_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_stop_server.yml')
GCLOUD_DELETE_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_delete_server.yml')
GCLOUD_CREATE_BUCKET_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_create_bucket.yml')
GCLOUD_SETUP_LOOM_USER_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_setup_loom_user.yml')
NGINX_CONFIG_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'nginx.conf'))

def ServerControlsFactory(args):
    """Factory method that checks ~/.loom/server.ini, then instantiates and returns the appropriate class."""

    # Check if we just need the base class (currently, it handles "set" and "status"):
    try:
        return BaseServerControls(args)
    except UnhandledCommandError:
        pass

    server_type = get_server_type()

    # Instantiate the appropriate subclass:
    if server_type == 'local':
        controls = LocalServerControls(args)
    elif server_type == 'gcloud':
        controls = GoogleCloudServerControls(args)
    else:
        raise Exception('Unrecognized server type: %s' % server_type)
    
    return controls

def get_parser(parser=None):
    if parser is None:
        parser = argparse.ArgumentParser(__file__)

    parser.add_argument('--test_database', '-t', action='store_true', help=argparse.SUPPRESS)
    parser.add_argument('--no_daemon', '-n', action='store_true', help=argparse.SUPPRESS)

    subparsers = parser.add_subparsers(dest='command')
    status_parser = subparsers.add_parser('status')

    create_parser = subparsers.add_parser('create')
    create_parser.add_argument('--require_default_settings', '-d', action='store_true', help=argparse.SUPPRESS)
    create_parser.add_argument('--settings', '-s', metavar='SETTINGS_FILE', default=None,
        help="A settings file can be provided to override default settings.")
    create_parser.add_argument('--verbose', '-v', action='store_true', help='Provide more feedback to console.')

    start_parser = subparsers.add_parser('start')
    start_parser.add_argument('--foreground', action='store_true', help='Run webserver in the foreground. Needed to keep Docker container running.')
    start_parser.add_argument('--verbose', '-v', action='store_true', help='Provide more feedback to console.')

    stop_parser = subparsers.add_parser('stop')
    stop_parser.add_argument('--verbose', '-v', action='store_true', help='Provide more feedback to console.')

    restart_parser = subparsers.add_parser('restart')
    restart_parser.add_argument('--foreground', action='store_true', help='Run webserver in the foreground. Needed to keep Docker container running.')
    restart_parser.add_argument('--verbose', '-v', action='store_true', help='Provide more feedback to console.')
    
    delete_parser = subparsers.add_parser('delete')
    delete_parser.add_argument('--verbose', '-v', action='store_true', help='Provide more feedback to console.')

    setserver_parser = subparsers.add_parser('set', help='Tells the client how to find the Loom server. This information is stored in %s.' % SERVER_LOCATION_FILE)
    setserver_parser.add_argument('type', choices=['local', 'gcloud'], help='The type of server the client will manage.')
    setserver_parser.add_argument('--name', help='Not used for local. For gcloud, the instance name of the server to manage. If not provided, defaults to %s.' % GCLOUD_SERVER_DEFAULT_NAME, metavar='GCE_INSTANCE_NAME', default=GCLOUD_SERVER_DEFAULT_NAME)

    return parser


class BaseServerControls:
    """Base class for managing the Loom server. Handles argument parsing, 
    subcommand routing, and implements common base methods such as setting
    the server type and checking the server status.
    """
    def __init__(self, args=None):
        if args is None:
            args=self._get_args()
        self.args = args
        self._set_run_function(args)

    # Defines what commands this class can handle and maps names to functions.
    def _get_command_map(self):
        return {
            'status': self.status,
            'set': self.setserver
        }

    def _set_run_function(self, args):
        # Map user input command to class method
        try:
            self.run = self._get_command_map()[args.command]
        except KeyError:
            raise UnhandledCommandError('Unrecognized command: %s' % args.command)

    def _get_args(self):
        parser = get_parser()
        args = parser.parse_args()
        return args

    def setserver(self):
        '''Set server for the client to manage (currently local or gcloud) and creates Loom settings directory.'''
        server_location_file = os.path.expanduser(SERVER_LOCATION_FILE)
        # Create directory/directories if they don't exist
        ini_dir = os.path.dirname(server_location_file)
        if not os.path.exists(ini_dir):
            print 'Creating Loom settings directory %s...' % ini_dir
            os.makedirs(ini_dir)

        # Write server.ini file
        config = SafeConfigParser()
        config.add_section('server')
        config.set('server', 'type', self.args.type)
        if self.args.type == 'gcloud':
            name = self.args.name
            config.set('server', 'name', name)
        with open(server_location_file, 'w') as configfile:
            print 'Updating %s...' % server_location_file
            config.write(configfile)

        # Copy NGINX config file to same place
        shutil.copy(NGINX_CONFIG_FILE, ini_dir)

    def status(self):
        if is_server_running():
            print 'OK. The server is running.'
        else:
            print 'No response for server at %s. Do you need to run "loom server start"?' % get_server_url()


class LocalServerControls(BaseServerControls):
    """Subclass for managing a server running on localhost."""

    def __init__(self, args=None):
        BaseServerControls.__init__(self, args)
        self.settings_manager = SettingsManager()

    # Defines what commands this class can handle and maps names to functions.
    def _get_command_map(self):
        command_to_method_map = {
            'create': self.create,
            'start': self.start,
            'restart': self.restart,
            'stop': self.stop,
            'delete': self.delete,
        }
        return command_to_method_map

    def start(self):
        if not is_server_running():
            if not os.path.exists(get_deploy_settings_filename()):
                self.create()
            self.settings_manager.load_deploy_settings_file()

            if loom.common.cloud.on_gcloud_vm():
                """We're in gcloud and the client is starting the server on the local instance."""
                subprocess.call(['ansible-playbook', GCLOUD_SETUP_LOOM_USER_PLAYBOOK])
                self.settings_manager.add_gcloud_settings_on_server()

            env = os.environ.copy()
            env = self._add_server_to_python_path(env)
            env = self._set_database(env)
            env = self._export_django_settings(env)
            self._create_logdirs()
            self._start_daemon(env)
            print 'Starting Loom server...'
            self._start_webserver(env)
            self._wait_for_server(target_running_state=True)
        print 'Loom server is running.'

    def stop(self):
        # Settings needed to get pidfile names.
        if is_server_running():
            if not os.path.exists(get_deploy_settings_filename()):
                raise Exception('No server deploy settings found. Create them with "loom server create" first.')
            self.settings_manager.load_deploy_settings_file()
            print "Stopping Loom server..."
            self._stop_webserver()
            self._stop_daemon()
            self._wait_for_server(target_running_state=False)
        print "Loom server is stopped."

    def restart(self):
        self.stop()
        self.start()

    def _wait_for_server(self, target_running_state=True):
        timeout_seconds=5
        poll_interval_seconds=0.1<commit_msg>apply settings before settings django database<commit_after>#!/usr/bin/env python

import argparse
from datetime import datetime
import imp
import os
import re
import shutil
import subprocess
import sys
import time
import warnings

from ConfigParser import SafeConfigParser
from loom.client.settings_manager import SettingsManager
from loom.client.common import *
from loom.common.version import version
import loom.common.cloud

DAEMON_EXECUTABLE = os.path.abspath(
    os.path.join(
    os.path.dirname(__file__),
    '../master/loomdaemon/loom_daemon.py'
    ))

GCLOUD_SERVER_DEFAULT_NAME = SettingsManager().get_default_setting('gcloud', 'SERVER_NAME')

PLAYBOOKS_PATH = os.path.join(imp.find_module('loom')[1], 'playbooks')
GCLOUD_CREATE_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_create_server.yml')
GCLOUD_START_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_start_server.yml')
GCLOUD_STOP_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_stop_server.yml')
GCLOUD_DELETE_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_delete_server.yml')
GCLOUD_CREATE_BUCKET_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_create_bucket.yml')
GCLOUD_SETUP_LOOM_USER_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_setup_loom_user.yml')
NGINX_CONFIG_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'nginx.conf'))

def ServerControlsFactory(args):
    """Factory method that checks ~/.loom/server.ini, then instantiates and returns the appropriate class."""

    # Check if we just need the base class (currently, it handles "set" and "status"):
    try:
        return BaseServerControls(args)
    except UnhandledCommandError:
        pass

    server_type = get_server_type()

    # Instantiate the appropriate subclass:
    if server_type == 'local':
        controls = LocalServerControls(args)
    elif server_type == 'gcloud':
        controls = GoogleCloudServerControls(args)
    else:
        raise Exception('Unrecognized server type: %s' % server_type)
    
    return controls

def get_parser(parser=None):
    if parser is None:
        parser = argparse.ArgumentParser(__file__)

    parser.add_argument('--test_database', '-t', action='store_true', help=argparse.SUPPRESS)
    parser.add_argument('--no_daemon', '-n', action='store_true', help=argparse.SUPPRESS)

    subparsers = parser.add_subparsers(dest='command')
    status_parser = subparsers.add_parser('status')

    create_parser = subparsers.add_parser('create')
    create_parser.add_argument('--require_default_settings', '-d', action='store_true', help=argparse.SUPPRESS)
    create_parser.add_argument('--settings', '-s', metavar='SETTINGS_FILE', default=None,
        help="A settings file can be provided to override default settings.")
    create_parser.add_argument('--verbose', '-v', action='store_true', help='Provide more feedback to console.')

    start_parser = subparsers.add_parser('start')
    start_parser.add_argument('--foreground', action='store_true', help='Run webserver in the foreground. Needed to keep Docker container running.')
    start_parser.add_argument('--verbose', '-v', action='store_true', help='Provide more feedback to console.')

    stop_parser = subparsers.add_parser('stop')
    stop_parser.add_argument('--verbose', '-v', action='store_true', help='Provide more feedback to console.')

    restart_parser = subparsers.add_parser('restart')
    restart_parser.add_argument('--foreground', action='store_true', help='Run webserver in the foreground. Needed to keep Docker container running.')
    restart_parser.add_argument('--verbose', '-v', action='store_true', help='Provide more feedback to console.')
    
    delete_parser = subparsers.add_parser('delete')
    delete_parser.add_argument('--verbose', '-v', action='store_true', help='Provide more feedback to console.')

    setserver_parser = subparsers.add_parser('set', help='Tells the client how to find the Loom server. This information is stored in %s.' % SERVER_LOCATION_FILE)
    setserver_parser.add_argument('type', choices=['local', 'gcloud'], help='The type of server the client will manage.')
    setserver_parser.add_argument('--name', help='Not used for local. For gcloud, the instance name of the server to manage. If not provided, defaults to %s.' % GCLOUD_SERVER_DEFAULT_NAME, metavar='GCE_INSTANCE_NAME', default=GCLOUD_SERVER_DEFAULT_NAME)

    return parser


class BaseServerControls:
    """Base class for managing the Loom server. Handles argument parsing, 
    subcommand routing, and implements common base methods such as setting
    the server type and checking the server status.
    """
    def __init__(self, args=None):
        if args is None:
            args=self._get_args()
        self.args = args
        self._set_run_function(args)

    # Defines what commands this class can handle and maps names to functions.
    def _get_command_map(self):
        return {
            'status': self.status,
            'set': self.setserver
        }

    def _set_run_function(self, args):
        # Map user input command to class method
        try:
            self.run = self._get_command_map()[args.command]
        except KeyError:
            raise UnhandledCommandError('Unrecognized command: %s' % args.command)

    def _get_args(self):
        parser = get_parser()
        args = parser.parse_args()
        return args

    def setserver(self):
        '''Set server for the client to manage (currently local or gcloud) and creates Loom settings directory.'''
        server_location_file = os.path.expanduser(SERVER_LOCATION_FILE)
        # Create directory/directories if they don't exist
        ini_dir = os.path.dirname(server_location_file)
        if not os.path.exists(ini_dir):
            print 'Creating Loom settings directory %s...' % ini_dir
            os.makedirs(ini_dir)

        # Write server.ini file
        config = SafeConfigParser()
        config.add_section('server')
        config.set('server', 'type', self.args.type)
        if self.args.type == 'gcloud':
            name = self.args.name
            config.set('server', 'name', name)
        with open(server_location_file, 'w') as configfile:
            print 'Updating %s...' % server_location_file
            config.write(configfile)

        # Copy NGINX config file to same place
        shutil.copy(NGINX_CONFIG_FILE, ini_dir)

    def status(self):
        if is_server_running():
            print 'OK. The server is running.'
        else:
            print 'No response for server at %s. Do you need to run "loom server start"?' % get_server_url()


class LocalServerControls(BaseServerControls):
    """Subclass for managing a server running on localhost."""

    def __init__(self, args=None):
        BaseServerControls.__init__(self, args)
        self.settings_manager = SettingsManager()

    # Defines what commands this class can handle and maps names to functions.
    def _get_command_map(self):
        command_to_method_map = {
            'create': self.create,
            'start': self.start,
            'restart': self.restart,
            'stop': self.stop,
            'delete': self.delete,
        }
        return command_to_method_map

    def start(self):
        if not is_server_running():
            if not os.path.exists(get_deploy_settings_filename()):
                self.create()
            self.settings_manager.load_deploy_settings_file()

            if loom.common.cloud.on_gcloud_vm():
                """We're in gcloud and the client is starting the server on the local instance."""
                subprocess.call(['ansible-playbook', GCLOUD_SETUP_LOOM_USER_PLAYBOOK])
                self.settings_manager.add_gcloud_settings_on_server()

            env = os.environ.copy()
            env = self._add_server_to_python_path(env)
            env = self._export_django_settings(env)
            env = self._set_database(env)
            self._create_logdirs()
            self._start_daemon(env)
            print 'Starting Loom server...'
            self._start_webserver(env)
            self._wait_for_server(target_running_state=True)
        print 'Loom server is running.'

    def stop(self):
        # Settings needed to get pidfile names.
        if is_server_running():
            if not os.path.exists(get_deploy_settings_filename()):
                raise Exception('No server deploy settings found. Create them with "loom server create" first.')
            self.settings_manager.load_deploy_settings_file()
            print "Stopping Loom server..."
            self._stop_webserver()
            self._stop_daemon()
            self._wait_for_server(target_running_state=False)
        print "Loom server is stopped."

    def restart(self):
        self.stop()
        self.start()

    def _wait_for_server(self, target_running_state=True):
        timeout_seconds=5
        poll_interval_seconds=0.1
{'fraction_non_alphanumeric': 0.10291382517048978, 'fraction_numerical': 0.0004508820379868117, 'mean_word_length': 3.2592414786365818, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1640662', 'n_tokens_mistral': 5248, 'n_tokens_neox': 5163, 'n_words': 1270}