<commit_before>from __future__ import absolute_import, division

import json
import logging
import requests
import time

from datetime import datetime
from more_itertools import peekable
from uuid import uuid4

from changes.backends.base import BaseBackend
from changes.config import db
from changes.constants import Result, Status
from changes.db.utils import create_or_update, get_or_create
from changes.models import RemoteEntity, TestResult, LogSource, LogChunk


def chunked(iterator, chunk_size):
    result = ''
    for chunk in iterator:
        result += chunk
        if len(result) > chunk_size:
            yield result[:chunk_size]
            result = result[chunk_size:]
    if result:
        yield result


class NotFound(Exception):
    pass


class JenkinsBuilder(BaseBackend):
    provider = 'jenkins'

    def __init__(self, base_url=None, token=None, *args, **kwargs):
        super(JenkinsBuilder, self).__init__(*args, **kwargs)
        self.base_url = base_url or self.app.config['JENKINS_URL']
        self.token = token or self.app.config['JENKINS_TOKEN']
        self.logger = logging.getLogger('jenkins')

    def _get_response(self, path, method='GET', params=None, **kwargs):
        url = '{}/{}/api/json/'.format(self.base_url, path.strip('/'))

        if params is None:
            params = {}

        params.setdefault('token', self.token)

        self.logger.info('Fetching %r', url)
        resp = getattr(requests, method.lower())(url, params=params, **kwargs)

        if resp.status_code == 404:
            raise NotFound
        elif not (200 <= resp.status_code < 300):
            raise Exception('Invalid response. Status code was %s' % resp.status_code)

        data = resp.text
        if data:
            try:
                return json.loads(data)
            except ValueError:
                raise Exception('Invalid JSON data')
        return

    def _parse_parameters(self, json):
        params = {}
        for action in json['actions']:
            params.update(
                (p['name'], p.get('value'))
                for p in action.get('parameters', [])
            )
        return params

    def _sync_build_from_queue(self, build, entity):
        # TODO(dcramer); determine whats going on w/ the JSON field
        build_item = entity.data.copy()

        # TODO(dcramer): when we hit a NotFound in the queue, maybe we should
        # attempt to scrape the list of jobs for a matching CHANGES_BID, as this
        # doesnt explicitly mean that the job doesnt exist
        try:
            item = self._get_response('/queue/item/{}'.format(
                build_item['item_id']))
        except NotFound:
            build.status = Status.finished
            build.result = Result.unknown
            db.session.add(build)
            return

        if item.get('executable'):
            build_no = item['executable']['number']
            build_item['queued'] = False
            build_item['build_no'] = build_no
            entity.data = build_item
            db.session.add(entity)

        if item['blocked']:
            build.status = Status.queued
            db.session.add(build)
        elif item.get('cancelled'):
            build.status = Status.finished
            build.result = Result.aborted
            db.session.add(build)
        elif item.get('executable'):
            for x in xrange(6):
                # There's a possible race condition where the item has been
                # assigned an ID, yet the API responds as if the build does
                # not exist
                try:
                    self._sync_build_from_active(build, entity, fail_on_404=True)
                except NotFound:
                    time.sleep(0.3)
                else:
                    break

    def _sync_build_from_active(self, build, entity, fail_on_404=False):
        build_item = entity.data.copy()
        try:
            item = self._get_response('/job/{}/{}'.format(
                build_item['job_name'], build_item['build_no']))
        except NotFound:
            if fail_on_404:
                raise
            build.date_finished = datetime.utcnow()
            build.status = Status.finished
            build.result = Result.aborted
            db.session.add(build)
            return

        should_finish = False

        # XXX(dcramer): timestamp implies creation date, so lets just assume
        # we were able to track it immediately
        if not build.date_started:
            build.date_started = datetime.utcnow()

        if item['building']:
            build.status = Status.in_progress
        else:
            should_finish = True
            build.date_finished = datetime.utcnow()

            if item['result'] == 'SUCCESS':
                build.result = Result.passed
            elif item['result'] == 'ABORTED':
                build.result = Result.aborted
            elif item['result'] in ('FAILURE', 'REGRESSION'):
                build.result = Result.failed
            else:
                raise ValueError('Invalid build result: %s' % (item['result'],))

        if item['duration']:
            build.duration = item['duration']

        build.data = {
            'backend': {
                'uri': item['url'],
                'label': item['fullDisplayName'],
            }
        }

        db.session.add(build)
        db.session.commit()

        # TODO(dcramer): ideally we could fire off jobs to sync test results
        # and console logs
        for action in item['actions']:
            # is this the best way to find this?
            if action.get('urlName') == 'testReport':
                self._sync_test_results(build, entity)
                break

        self._sync_console_log(build, entity)

        if should_finish:
            build.status = Status.finished
            db.session.add(build)

    def _sync_console_log(self, build, entity):
        # TODO(dcramer): this doesnt handle concurrency
        build_item = entity.data
        logsource, created = get_or_create(LogSource, where={
            'name': 'console',
            'build': build,
        }, defaults={
            'project': build.project,
            'date_created': build.date_started,
        })
        if created:
            offset = 0
        else:
            offset = build_item.get('log_offset', 0)
<commit_msg>Dont sync logs until build finishes<commit_after>from __future__ import absolute_import, division

import json
import logging
import requests
import time

from datetime import datetime
from more_itertools import peekable
from uuid import uuid4

from changes.backends.base import BaseBackend
from changes.config import db
from changes.constants import Result, Status
from changes.db.utils import create_or_update, get_or_create
from changes.models import RemoteEntity, TestResult, LogSource, LogChunk


def chunked(iterator, chunk_size):
    result = ''
    for chunk in iterator:
        result += chunk
        if len(result) > chunk_size:
            yield result[:chunk_size]
            result = result[chunk_size:]
    if result:
        yield result


class NotFound(Exception):
    pass


class JenkinsBuilder(BaseBackend):
    provider = 'jenkins'

    def __init__(self, base_url=None, token=None, *args, **kwargs):
        super(JenkinsBuilder, self).__init__(*args, **kwargs)
        self.base_url = base_url or self.app.config['JENKINS_URL']
        self.token = token or self.app.config['JENKINS_TOKEN']
        self.logger = logging.getLogger('jenkins')

    def _get_response(self, path, method='GET', params=None, **kwargs):
        url = '{}/{}/api/json/'.format(self.base_url, path.strip('/'))

        if params is None:
            params = {}

        params.setdefault('token', self.token)

        self.logger.info('Fetching %r', url)
        resp = getattr(requests, method.lower())(url, params=params, **kwargs)

        if resp.status_code == 404:
            raise NotFound
        elif not (200 <= resp.status_code < 300):
            raise Exception('Invalid response. Status code was %s' % resp.status_code)

        data = resp.text
        if data:
            try:
                return json.loads(data)
            except ValueError:
                raise Exception('Invalid JSON data')
        return

    def _parse_parameters(self, json):
        params = {}
        for action in json['actions']:
            params.update(
                (p['name'], p.get('value'))
                for p in action.get('parameters', [])
            )
        return params

    def _sync_build_from_queue(self, build, entity):
        # TODO(dcramer); determine whats going on w/ the JSON field
        build_item = entity.data.copy()

        # TODO(dcramer): when we hit a NotFound in the queue, maybe we should
        # attempt to scrape the list of jobs for a matching CHANGES_BID, as this
        # doesnt explicitly mean that the job doesnt exist
        try:
            item = self._get_response('/queue/item/{}'.format(
                build_item['item_id']))
        except NotFound:
            build.status = Status.finished
            build.result = Result.unknown
            db.session.add(build)
            return

        if item.get('executable'):
            build_no = item['executable']['number']
            build_item['queued'] = False
            build_item['build_no'] = build_no
            entity.data = build_item
            db.session.add(entity)

        if item['blocked']:
            build.status = Status.queued
            db.session.add(build)
        elif item.get('cancelled'):
            build.status = Status.finished
            build.result = Result.aborted
            db.session.add(build)
        elif item.get('executable'):
            for x in xrange(6):
                # There's a possible race condition where the item has been
                # assigned an ID, yet the API responds as if the build does
                # not exist
                try:
                    self._sync_build_from_active(build, entity, fail_on_404=True)
                except NotFound:
                    time.sleep(0.3)
                else:
                    break

    def _sync_build_from_active(self, build, entity, fail_on_404=False):
        build_item = entity.data.copy()
        try:
            item = self._get_response('/job/{}/{}'.format(
                build_item['job_name'], build_item['build_no']))
        except NotFound:
            if fail_on_404:
                raise
            build.date_finished = datetime.utcnow()
            build.status = Status.finished
            build.result = Result.aborted
            db.session.add(build)
            return

        should_finish = False

        # XXX(dcramer): timestamp implies creation date, so lets just assume
        # we were able to track it immediately
        if not build.date_started:
            build.date_started = datetime.utcnow()

        if item['building']:
            build.status = Status.in_progress
        else:
            should_finish = True
            build.date_finished = datetime.utcnow()

            if item['result'] == 'SUCCESS':
                build.result = Result.passed
            elif item['result'] == 'ABORTED':
                build.result = Result.aborted
            elif item['result'] in ('FAILURE', 'REGRESSION'):
                build.result = Result.failed
            else:
                raise ValueError('Invalid build result: %s' % (item['result'],))

        if item['duration']:
            build.duration = item['duration']

        build.data = {
            'backend': {
                'uri': item['url'],
                'label': item['fullDisplayName'],
            }
        }

        db.session.add(build)
        db.session.commit()

        # TODO(dcramer): ideally we could fire off jobs to sync test results
        # and console logs
        for action in item['actions']:
            # is this the best way to find this?
            if action.get('urlName') == 'testReport':
                self._sync_test_results(build, entity)
                break

        if should_finish:
            # FIXME(dcramer): we're waiting until the build is complete to sync
            # logs due to our inability to correctly identify start offsets
            self._sync_console_log(build, entity)

            build.status = Status.finished
            db.session.add(build)

    def _sync_console_log(self, build, entity):
        # TODO(dcramer): this doesnt handle concurrency
        build_item = entity.data
        logsource, created = get_or_create(LogSource, where={
            'name': 'console',
            'build': build,
        }, defaults={
            'project': build.project,
            'date_created': build.date_started,
        })
        if created:
            offset = 0
        else:
            offset = build_item.get('log_offset', 0)

{'fraction_non_alphanumeric': 0.09893502083654886, 'fraction_numerical': 0.0037042753511344344, 'mean_word_length': 1.891343150379295, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '229608', 'n_tokens_mistral': 3537, 'n_tokens_neox': 3380, 'n_words': 937}