<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
gitdir autodeploy
"""

import bottle
import datetime
import gitdir.repo
import json
import os.path
import subprocess
import traceback

try:
    import uwsgi
    CONFIG_PATH = uwsgi.opt['config_path']
except:
    CONFIG_PATH = '/etc/xdg/gitdir/autodeploy.json'

bottle.debug()

application = bottle.Bottle()

try:
    with open(CONFIG_PATH) as config_file:
        config = json.load(config_file)
except:
    config = {}

class DeployError(Exception):
    pass

def try_subprocess(args, cwd=None, request_time=None):
    if request_time is None:
        request_time = datetime.datetime.utcnow()
    popen = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd)
    if popen.wait() != 0:
        if 'logPath' in config and os.path.exists(config['logPath']):
            with open(os.path.join(config['logPath'], request_time.strftime('%Y%m%d-%H%M%S-%f-error.log')), 'a') as log_file:
                print('subprocess output:', file=log_file)
                print(popen.stdout.read().decode('utf-8'), file=log_file)
        raise subprocess.CalledProcessError(popen.returncode, args)

@application.route('/deploy/<hostname>/<repo_spec:path>/<branch>')
def deploy(hostname, repo_spec, branch='master', request_time=None):
    try:
        gitdir.repo.by_name(hostname).deploy(repo_spec, branch=branch)
    except Exception as e:
        if 'logPath' in config and os.path.exists(config['logPath']) and request_time is not None:
            with open(os.path.join(config['logPath'], request_time.strftime('%Y%m%d-%H%M%S-%f-error.log')), 'a') as log_file:
                print('Error while deploying from {}:'.format(hostname), file=log_file)                print('username: ' + user, file=log_file)
                print('repo: ' + repo_spec, file=log_file)
                print('branch: ' + (branch or 'master'), file=log_file)
                traceback.print_exc(file=log_file)
        raise DeployError('{} while deploying. host: {!r}, user: {!r}, repo: {!r}, branch: {!r}'.format(e.__class__.__name__, hostname, repo_spec, branch)) from e

@application.route('/')
def show_index():
    return bottle.static_file('static/index.html', root=config.get('documentRoot', os.path.dirname(os.path.abspath(__file__))))

@application.route('/deploy')
def get_deploy():
    request_time = datetime.datetime.utcnow()
    for host, host_data in config.get('repos', {}).items():
        for repo_spec, branches in host_data.items():
            for branch in branches:
                deploy(host, repo_spec, branch=branch, request_time=request_time)

@application.route('/deploy', method='POST')
def post_deploy():
    if 'logPath' in config and os.path.exists(config['logPath']):
        with open(os.path.join(config['logPath'], datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S-%f.json')), 'w') as log_file:
            json.dump(bottle.request.json, log_file, sort_keys=True, indent=4, separators=(',', ': '))
    return get_deploy() #TODO read the payload to see what to deploy

if __name__ == '__main__':
    bottle.run(app=application, host='0.0.0.0', port=8081)<commit_msg>Change config file structure to allow room for repo-specific options<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
gitdir autodeploy
"""

import bottle
import datetime
import gitdir.repo
import json
import os.path
import subprocess
import traceback

try:
    import uwsgi
    CONFIG_PATH = uwsgi.opt['config_path']
except:
    CONFIG_PATH = '/etc/xdg/gitdir/autodeploy.json'

bottle.debug()

application = bottle.Bottle()

try:
    with open(CONFIG_PATH) as config_file:
        config = json.load(config_file)
except:
    config = {}

class DeployError(Exception):
    pass

def try_subprocess(args, cwd=None, request_time=None):
    if request_time is None:
        request_time = datetime.datetime.utcnow()
    popen = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd)
    if popen.wait() != 0:
        if 'logPath' in config and os.path.exists(config['logPath']):
            with open(os.path.join(config['logPath'], request_time.strftime('%Y%m%d-%H%M%S-%f-error.log')), 'a') as log_file:
                print('subprocess output:', file=log_file)
                print(popen.stdout.read().decode('utf-8'), file=log_file)
        raise subprocess.CalledProcessError(popen.returncode, args)

@application.route('/deploy/<hostname>/<repo_spec:path>/<branch>')
def deploy(hostname, repo_spec, branch='master', request_time=None):
    try:
        gitdir.repo.by_name(hostname).deploy(repo_spec, branch=branch)
    except Exception as e:
        if 'logPath' in config and os.path.exists(config['logPath']) and request_time is not None:
            with open(os.path.join(config['logPath'], request_time.strftime('%Y%m%d-%H%M%S-%f-error.log')), 'a') as log_file:
                print('Error while deploying from {}:'.format(hostname), file=log_file)                print('username: ' + user, file=log_file)
                print('repo: ' + repo_spec, file=log_file)
                print('branch: ' + (branch or 'master'), file=log_file)
                traceback.print_exc(file=log_file)
        raise DeployError('{} while deploying. host: {!r}, user: {!r}, repo: {!r}, branch: {!r}'.format(e.__class__.__name__, hostname, repo_spec, branch)) from e

@application.route('/')
def show_index():
    return bottle.static_file('static/index.html', root=config.get('documentRoot', os.path.dirname(os.path.abspath(__file__))))

@application.route('/deploy')
def get_deploy():
    request_time = datetime.datetime.utcnow()
    for host, host_data in config.get('repos', {}).items():
        for repo_spec, repo_info in host_data.items():
            for branch in repo_info.get('branches', ['master']):
                deploy(host, repo_spec, branch=branch, request_time=request_time)

@application.route('/deploy', method='POST')
def post_deploy():
    if 'logPath' in config and os.path.exists(config['logPath']):
        with open(os.path.join(config['logPath'], datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S-%f.json')), 'w') as log_file:
            json.dump(bottle.request.json, log_file, sort_keys=True, indent=4, separators=(',', ': '))
    return get_deploy() #TODO read the payload to see what to deploy

if __name__ == '__main__':
    bottle.run(app=application, host='0.0.0.0', port=8081)
{'fraction_non_alphanumeric': 0.1559432933478735, 'fraction_numerical': 0.004050475151892818, 'mean_word_length': 3.7874720357941833, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3162887', 'n_tokens_mistral': 2155, 'n_tokens_neox': 2113, 'n_words': 439}