<commit_before>import sys
import os
import subprocess
import threading
import urllib2
import BaseHTTPServer
import SimpleHTTPServer
import json
import yaml

from irc import ircbot

DEBUG = False


class MyHTTPServer(BaseHTTPServer.HTTPServer):
    allow_reuse_address = True


class MyHTTPServerHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    
    locks = {}
    
    @staticmethod
    def get_lock(ident):
        return MyHTTPServerHandler.locks.get(ident, threading.Semaphore())
    
    def do_POST(self):
        content_len = int(self.headers.getheader('content-length'))
        post_body = self.rfile.read(content_len)
        if post_body.startswith("payload"):
            json_payload = json.loads(urllib2.unquote(post_body[8:]))
            data = parse(json_payload)
            ident = '{0}/{1}'.format(data["name"], data["repo_name"])
            t = threading.Thread(target=handle_hook, args=(ident, data))
            t.start()

def run_server():
    httpd = MyHTTPServer(('', 13373), MyHTTPServerHandler)
    httpd.serve_forever()

def parse(json):
    """Parse the received JSON and return a dict of the valuable information."""
    try:
        repo_name = json["repository"]["name"]
        owner_name = json["repository"]["owner"]["name"]
        push_name = json["pusher"]["name"]
        commits = str(len(json["commits"]))
        ref = json["ref"]
        url = json["repository"]["url"]
    except IndexError:
        return None
    else:
        return {
            "name": owner_name, "repo_name": repo_name,
            "push_name": push_name, "commits": commits,
            "ref": ref, "url": url
        }

def handle_hook(ident, data):
    """
    Manage the different actions that need to be taken once a hook is received.
    Semaphores are used to make sure two pushs in quick succession doesn't happen
    simultaneously, but, rather sequentially.
     
    """
    # Only execute on push to master
    sem = MyHTTPServerHandler.get_lock(ident)
    sem.acquire()
    if data is not None and data["ref"] == "refs/heads/master":
        check_repository_existence(data)
        git_update(data)
        execute_shell_script(data)
    sem.release()

def check_repository_existence(data):
    """
    Check if a repository exists (and the subsequent owner), else, clone
    down the repository from the url received with the hook.
    
    """
    owner_dir = 'repos/{0}'.format(data["name"])
    if not os.path.isdir(owner_dir):
        os.mkdir(owner_dir)
    repo_dir = 'repos/{0}/{1}'.format(data["name"], data["repo_name"])
    if not os.path.isdir(repo_dir):
        notify('Cloning `{0}/{1}`...'.format(data["name"], data["repo_name"]))
        with open(os.devnull, 'w') as devnull:
            subprocess.Popen(
                ['git', 'clone', data["url"], repo_dir],
                stdout=devnull, stderr=devnull
            )
    
def git_update(data):
    """Perform a git pull in the repository."""
    #notify('Hook received by `{0}` for `{1}/{2}`, performing pull...'.format(
    #    data["push_name"], data["name"], data["repo_name"]
    #))
    repo_dir = 'repos/{0}/{1}'.format(data["name"], data["repo_name"])
    with open(os.devnull, 'w') as devnull:
        subprocess.Popen(
            ['git', 'pull', 'origin', 'master'],
            cwd=repo_dir, stdout=devnull, stderr=devnull
        )
    notify('Updated `{0}/{1}` with {2} commit(s)!'.format(
        data["name"], data["repo_name"], data["commits"]
    ))
    
def execute_shell_script(data):
    """Execute the accompanying shell/deploy script for the repo."""
    script = 'scripts/{0}_{1}'.format(data["name"].lower(), data["repo_name"].lower())
    if os.path.exists(script):<commit_msg>Add support for submodules.<commit_after>import sys
import os
import subprocess
import threading
import urllib2
import BaseHTTPServer
import SimpleHTTPServer
import json
import yaml

from irc import ircbot

DEBUG = False


class MyHTTPServer(BaseHTTPServer.HTTPServer):
    allow_reuse_address = True


class MyHTTPServerHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    
    locks = {}
    
    @staticmethod
    def get_lock(ident):
        return MyHTTPServerHandler.locks.get(ident, threading.Semaphore())
    
    def do_POST(self):
        content_len = int(self.headers.getheader('content-length'))
        post_body = self.rfile.read(content_len)
        if post_body.startswith("payload"):
            json_payload = json.loads(urllib2.unquote(post_body[8:]))
            data = parse(json_payload)
            ident = '{0}/{1}'.format(data["name"], data["repo_name"])
            t = threading.Thread(target=handle_hook, args=(ident, data))
            t.start()

def run_server():
    httpd = MyHTTPServer(('', 13373), MyHTTPServerHandler)
    httpd.serve_forever()

def parse(json):
    """Parse the received JSON and return a dict of the valuable information."""
    try:
        repo_name = json["repository"]["name"]
        owner_name = json["repository"]["owner"]["name"]
        push_name = json["pusher"]["name"]
        commits = str(len(json["commits"]))
        ref = json["ref"]
        url = json["repository"]["url"]
    except IndexError:
        return None
    else:
        return {
            "name": owner_name, "repo_name": repo_name,
            "push_name": push_name, "commits": commits,
            "ref": ref, "url": url
        }

def handle_hook(ident, data):
    """
    Manage the different actions that need to be taken once a hook is received.
    Semaphores are used to make sure two pushs in quick succession doesn't happen
    simultaneously, but, rather sequentially.
     
    """
    # Only execute on push to master
    sem = MyHTTPServerHandler.get_lock(ident)
    sem.acquire()
    if data is not None and data["ref"] == "refs/heads/master":
        check_repository_existence(data)
        git_update(data)
        execute_shell_script(data)
    sem.release()

def check_repository_existence(data):
    """
    Check if a repository exists (and the subsequent owner), else, clone
    down the repository from the url received with the hook.
    
    """
    owner_dir = 'repos/{0}'.format(data["name"])
    if not os.path.isdir(owner_dir):
        os.mkdir(owner_dir)
    repo_dir = 'repos/{0}/{1}'.format(data["name"], data["repo_name"])
    if not os.path.isdir(repo_dir):
        notify('Cloning `{0}/{1}`...'.format(data["name"], data["repo_name"]))
        with open(os.devnull, 'w') as devnull:
            subprocess.Popen(
                ['git', 'clone', data["url"], repo_dir],
                stdout=devnull, stderr=devnull
            )
    
def git_update(data):
    """Perform a git pull in the repository."""
    #notify('Hook received by `{0}` for `{1}/{2}`, performing pull...'.format(
    #    data["push_name"], data["name"], data["repo_name"]
    #))
    repo_dir = 'repos/{0}/{1}'.format(data["name"], data["repo_name"])
    with open(os.devnull, 'w') as devnull:
        subprocess.Popen(
            ['git', 'pull', 'origin', 'master'],
            cwd=repo_dir, stdout=devnull, stderr=devnull
        )
        subprocess.Popen(
            ['git', 'submodule', 'init'],
            cwd=repo_dir, stdout=devnull, stderr=devnull
        )
        subprocess.Popen(
            ['git', 'submodule', 'update'],
            cwd=repo_dir, stdout=devnull, stderr=devnull
        )
    notify('Updated `{0}/{1}` with {2} commit(s)!'.format(
        data["name"], data["repo_name"], data["commits"]
    ))
    
def execute_shell_script(data):
    """Execute the accompanying shell/deploy script for the repo."""
    script = 'scripts/{0}_{1}'.format(data["name"].lower(), data["repo_name"].lower())
    if os.path.exists(script):
{'fraction_non_alphanumeric': 0.1439865215137377, 'fraction_numerical': 0.006480041472265423, 'mean_word_length': 2.9192483494159474, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3970792', 'n_tokens_mistral': 2393, 'n_tokens_neox': 2398, 'n_words': 562}