<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vi:ts=4:softtabstop=4:shiftwidth=4

import ConfigParser
import argparse
import Queue
import json
import threading
import time

import os
import sys
import subprocess
import shutil
import yaml

import logging
import logging.handlers

from os.path import basename, dirname
from resource import getrlimit, RLIMIT_NOFILE

from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
import cgi

from contextlib import contextmanager

import daemon
import lockfile
try:
    from lockfile import pidlockfile
except ImportError:
    from daemon import pidlockfile

import tempfile
import stat

import paramiko

GIT_SSH_WRAPPER = '''#!/bin/sh

if [ -z "$PKEY" ]; then
    # if PKEY is not specified, run ssh using default keyfile
    ssh "$@"
else
    ssh -i "$PKEY" "$@"
fi
'''

class WatchGerrit(threading.Thread):
    """ Threaded job; listens for Gerrit events and puts them in a queue """

    def __init__(self, options, queue):
        options['port'] = int(options['port']) # convert to int?
        if 'timeout' not in options:
            options['timeout'] = 60
        self.options = options
        self.queue = queue
        self.logger = logging.getLogger('RepoWatch.WatchGerrit')
        threading.Thread.__init__(self)

    def get_extra(self, project):
        """ Fetch list of extra refs to check out.

            returns tuple (ref, change_$number)

        """
        extra_refs = []
        try:
            client = paramiko.SSHClient()
            client.load_system_host_keys()
            client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            client.connect(**self.options)
            _, stdout, _ = client.exec_command('gerrit query "status:open project:{0}" --patch-sets --format json'.format(project))
            for line in stdout:
                # get
                data = json.loads(line)
                if data.get('status', None):
                    extra_refs.append([data['patchSets'][-1:][0]['ref'], 'change_{0}'.format(data['number'])])

        except Exception, e:
            self.logger.exception('get_extra error: {0}'.format(str(e)))
        finally:
            client.close()

        self.logger.debug('Adding extra Gerrit refs: {0}'.format(extra_refs))
        return extra_refs

    def run(self):
        while 1:

            try:
                client = paramiko.SSHClient()
                client.load_system_host_keys()
                client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
                client.connect(**self.options)
                client.get_transport().set_keepalive(60)
                _, stdout, _ = client.exec_command('gerrit stream-events')
                for line in stdout:
                    #self.queue.put(json.loads(line))
                    self.handle_event(json.loads(line))
            except Exception, e:
                logging.exception('WatchGerrit: error: {0}'.format(str(e)))
            finally:
                client.close()
            time.sleep(5)

    def handle_event(self, event):
        try:
            event_project = event['change']['project']
        except KeyError:
            event_project = event['refUpdate']['project']

        self.logger.debug('Gerrit event: {0}'.format(event))

        # for all patchsets and drafts we handle those as special
        if event['type'] in ['patchset-created',
                             'draft-published',
                             'change-restored',
                             'comment-added']:
            self.queue.put({'type': 'update',
                            'project_name': event['change']['project'],
                            'branch_name': event['patchSet']['ref'],
                            'output_dir': 'change_{0}'.format(basename(dirname(event['patchSet']['ref'])))})

        # need to remove the branch_name directory that was created, a change-merged
        # also triggers a ref-updated event
        if event['type'] in ['change-abandoned',
                             'change-merged']:
            self.queue.put({'type': 'delete',
                            'project_name': event['change']['project'],
                            'branch_name': 'change_{0}'.format(basename(dirname(event['patchSet']['ref'])))})

        # for ref updates, this needs to handle creating and deleting branch_namees and updating
        if event['type'] in ['ref-updated']:
            if event['refUpdate']['newRev'] == u'0000000000000000000000000000000000000000':<commit_msg>remove comment-added event<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vi:ts=4:softtabstop=4:shiftwidth=4

import ConfigParser
import argparse
import Queue
import json
import threading
import time

import os
import sys
import subprocess
import shutil
import yaml

import logging
import logging.handlers

from os.path import basename, dirname
from resource import getrlimit, RLIMIT_NOFILE

from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
import cgi

from contextlib import contextmanager

import daemon
import lockfile
try:
    from lockfile import pidlockfile
except ImportError:
    from daemon import pidlockfile

import tempfile
import stat

import paramiko

GIT_SSH_WRAPPER = '''#!/bin/sh

if [ -z "$PKEY" ]; then
    # if PKEY is not specified, run ssh using default keyfile
    ssh "$@"
else
    ssh -i "$PKEY" "$@"
fi
'''

class WatchGerrit(threading.Thread):
    """ Threaded job; listens for Gerrit events and puts them in a queue """

    def __init__(self, options, queue):
        options['port'] = int(options['port']) # convert to int?
        if 'timeout' not in options:
            options['timeout'] = 60
        self.options = options
        self.queue = queue
        self.logger = logging.getLogger('RepoWatch.WatchGerrit')
        threading.Thread.__init__(self)

    def get_extra(self, project):
        """ Fetch list of extra refs to check out.

            returns tuple (ref, change_$number)

        """
        extra_refs = []
        try:
            client = paramiko.SSHClient()
            client.load_system_host_keys()
            client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            client.connect(**self.options)
            _, stdout, _ = client.exec_command('gerrit query "status:open project:{0}" --patch-sets --format json'.format(project))
            for line in stdout:
                # get
                data = json.loads(line)
                if data.get('status', None):
                    extra_refs.append([data['patchSets'][-1:][0]['ref'], 'change_{0}'.format(data['number'])])

        except Exception, e:
            self.logger.exception('get_extra error: {0}'.format(str(e)))
        finally:
            client.close()

        self.logger.debug('Adding extra Gerrit refs: {0}'.format(extra_refs))
        return extra_refs

    def run(self):
        while 1:

            try:
                client = paramiko.SSHClient()
                client.load_system_host_keys()
                client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
                client.connect(**self.options)
                client.get_transport().set_keepalive(60)
                _, stdout, _ = client.exec_command('gerrit stream-events')
                for line in stdout:
                    #self.queue.put(json.loads(line))
                    self.handle_event(json.loads(line))
            except Exception, e:
                logging.exception('WatchGerrit: error: {0}'.format(str(e)))
            finally:
                client.close()
            time.sleep(5)

    def handle_event(self, event):
        try:
            event_project = event['change']['project']
        except KeyError:
            event_project = event['refUpdate']['project']

        self.logger.debug('Gerrit event: {0}'.format(event))

        # for all patchsets and drafts we handle those as special
        if event['type'] in ['patchset-created',
                             'draft-published',
                             'change-restored']:
            self.queue.put({'type': 'update',
                            'project_name': event['change']['project'],
                            'branch_name': event['patchSet']['ref'],
                            'output_dir': 'change_{0}'.format(basename(dirname(event['patchSet']['ref'])))})

        # need to remove the branch_name directory that was created, a change-merged
        # also triggers a ref-updated event
        if event['type'] in ['change-abandoned',
                             'change-merged']:
            self.queue.put({'type': 'delete',
                            'project_name': event['change']['project'],
                            'branch_name': 'change_{0}'.format(basename(dirname(event['patchSet']['ref'])))})

        # for ref updates, this needs to handle creating and deleting branch_namees and updating
        if event['type'] in ['ref-updated']:
            if event['refUpdate']['newRev'] == u'0000000000000000000000000000000000000000':
{'fraction_non_alphanumeric': 0.11782210955582498, 'fraction_numerical': 0.013226055328998126, 'mean_word_length': 2.2074938140685756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6163768', 'n_tokens_mistral': 2611, 'n_tokens_neox': 2432, 'n_words': 606}