<commit_before>from __future__ import absolute_import, division

import json
import requests
import sys

from collections import defaultdict
from datetime import datetime, timedelta

from changes.backends.base import BaseBackend, UnrecoverableException
from changes.config import db
from changes.constants import Result, Status
from changes.models import JobPhase, JobStep, RemoteEntity, Node


class KoalityBuilder(BaseBackend):
    provider = 'koality'

    def __init__(self, base_url=None, api_key=None, project_id=None,
                 *args, **kwargs):
        super(KoalityBuilder, self).__init__(*args, **kwargs)
        self.base_url = base_url or self.app.config['KOALITY_URL']
        self.api_key = api_key or self.app.config['KOALITY_API_KEY']
        self.project_id = project_id
        self._node_cache = {}

    def _get_end_time(self, stage_list):
        end_time = 0
        for stage in stage_list:
            if not stage.get('endTime'):
                return
            end_time = max((stage['endTime'], end_time))

        if end_time != 0:
            return datetime.utcfromtimestamp(end_time / 1000)
        return

    def _get_start_time(self, stage_list):
        start_time = sys.maxint
        for stage in stage_list:
            if not stage.get('startTime'):
                continue
            start_time = min((stage['startTime'], start_time))

        if start_time not in (0, sys.maxint):
            return datetime.utcfromtimestamp(start_time / 1000)
        return

        datetime.utcfromtimestamp(
            min(s['startTime'] for s in stage_list if s['startTime']) / 1000,
        )

    def _get_response(self, method, url, **kwargs):
        kwargs.setdefault('params', {})
        # TODO(dcramer): ensure SSL is usable
        kwargs.setdefault('verify', False)
        kwargs['params'].setdefault('key', self.api_key)
        response = getattr(requests, method.lower())(url, **kwargs)
        data = response.text
        try:
            return json.loads(data)
        except ValueError:
            raise Exception(data)

    def _get_node(self, node_id):
        node = self._node_cache.get(node_id)
        if node is not None:
            return node

        entity = RemoteEntity.query.filter_by(
            type='node',
            remote_id=str(node_id),
            provider=self.provider,
        ).first()
        if entity is None:
            node = Node()
            entity = RemoteEntity(
                provider=self.provider, remote_id=str(node_id),
                internal_id=node.id, type='node',
            )
            db.session.add(node)
            db.session.add(entity)
        else:
            node = Node.query.get(entity.internal_id)

        self._node_cache[node_id] = node

        return node

    def _sync_phase(self, job, stage_type, stage_list, phase=None):
        remote_id = '%s:%s' % (job.id.hex, stage_type)

        if phase is None:
            entity = RemoteEntity.query.filter_by(
                type='phase',
                provider=self.provider,
                remote_id=remote_id,
            ).first()
            if entity is None:
                phase = None
                create_entity = True
            else:
                phase = JobPhase.query.get(entity.internal_id)
                create_entity = False
        else:
            create_entity = False

        if phase is None:
            phase = JobPhase()

        phase.job = job
        phase.repository = job.repository
        phase.project = job.project
        phase.label = stage_type.title()

        phase.date_started = self._get_start_time(stage_list)
        phase.date_finished = self._get_end_time(stage_list)

        # for stage in (s for s in stages if s['status'] == 'failed'):
        if phase.date_started and phase.date_finished:
            if all(s['status'] == 'passed' for s in stage_list):
                phase.result = Result.passed
            else:
                phase.result = Result.failed
            phase.status = Status.finished
        elif phase.date_started:
            if any(s['status'] == 'failed' for s in stage_list):
                phase.result = Result.failed
            else:
                phase.result = Result.unknown
            phase.status = Status.in_progress
        else:
            phase.status = Status.queued
            phase.result = Result.unknown

        if create_entity:
            entity = RemoteEntity(
                provider=self.provider, remote_id=remote_id,
                internal_id=phase.id, type='phase',
            )
            db.session.add(entity)

        db.session.add(phase)

        return phase

    def _sync_step(self, job, phase, stage, step=None):
        if step is None:
            try:
                entity = RemoteEntity.query.filter_by(
                    type='step',
                    provider=self.provider,
                    remote_id=str(stage['id']),
                )[0]
            except IndexError:
                step, entity = None, None
            else:
                step = JobStep.query.get(entity.internal_id)
            create_entity = entity is None
        else:
            create_entity = False

        if step is None:
            step = JobStep()

        node = self._get_node(stage['buildNode'])

        step.job = job
        step.repository = job.repository
        step.project = job.project
        step.phase = phase
        step.node = node
        step.label = stage['name']
        step.date_started = self._get_start_time([stage])
        step.date_finished = self._get_end_time([stage])

        if step.date_started and step.date_finished:
            if stage['status'] == 'passed':
                step.result = Result.passed
            else:
                step.result = Result.failed
            step.status = Status.finished
        elif step.date_started:
            if stage['status'] == 'failed':
                step.result = Result.failed
            else:
                step.result = Result.unknown
            step.status = Status.in_progress
        else:
            step.status = Status.queued
            step.result = Result.unknown

        if create_entity:
            entity = RemoteEntity(
                provider=self.provider, remote_id=str(stage['id']),
                internal_id=step.id, type='step',
            )
            db.session.add(entity)

        db.session.add(step)<commit_msg>Use ids rather than relationships<commit_after>from __future__ import absolute_import, division

import json
import requests
import sys

from collections import defaultdict
from datetime import datetime, timedelta

from changes.backends.base import BaseBackend, UnrecoverableException
from changes.config import db
from changes.constants import Result, Status
from changes.models import JobPhase, JobStep, RemoteEntity, Node


class KoalityBuilder(BaseBackend):
    provider = 'koality'

    def __init__(self, base_url=None, api_key=None, project_id=None,
                 *args, **kwargs):
        super(KoalityBuilder, self).__init__(*args, **kwargs)
        self.base_url = base_url or self.app.config['KOALITY_URL']
        self.api_key = api_key or self.app.config['KOALITY_API_KEY']
        self.project_id = project_id
        self._node_cache = {}

    def _get_end_time(self, stage_list):
        end_time = 0
        for stage in stage_list:
            if not stage.get('endTime'):
                return
            end_time = max((stage['endTime'], end_time))

        if end_time != 0:
            return datetime.utcfromtimestamp(end_time / 1000)
        return

    def _get_start_time(self, stage_list):
        start_time = sys.maxint
        for stage in stage_list:
            if not stage.get('startTime'):
                continue
            start_time = min((stage['startTime'], start_time))

        if start_time not in (0, sys.maxint):
            return datetime.utcfromtimestamp(start_time / 1000)
        return

        datetime.utcfromtimestamp(
            min(s['startTime'] for s in stage_list if s['startTime']) / 1000,
        )

    def _get_response(self, method, url, **kwargs):
        kwargs.setdefault('params', {})
        # TODO(dcramer): ensure SSL is usable
        kwargs.setdefault('verify', False)
        kwargs['params'].setdefault('key', self.api_key)
        response = getattr(requests, method.lower())(url, **kwargs)
        data = response.text
        try:
            return json.loads(data)
        except ValueError:
            raise Exception(data)

    def _get_node(self, node_id):
        node = self._node_cache.get(node_id)
        if node is not None:
            return node

        entity = RemoteEntity.query.filter_by(
            type='node',
            remote_id=str(node_id),
            provider=self.provider,
        ).first()
        if entity is None:
            node = Node()
            entity = RemoteEntity(
                provider=self.provider, remote_id=str(node_id),
                internal_id=node.id, type='node',
            )
            db.session.add(node)
            db.session.add(entity)
        else:
            node = Node.query.get(entity.internal_id)

        self._node_cache[node_id] = node

        return node

    def _sync_phase(self, job, stage_type, stage_list, phase=None):
        remote_id = '%s:%s' % (job.id.hex, stage_type)

        if phase is None:
            entity = RemoteEntity.query.filter_by(
                type='phase',
                provider=self.provider,
                remote_id=remote_id,
            ).first()
            if entity is None:
                phase = None
                create_entity = True
            else:
                phase = JobPhase.query.get(entity.internal_id)
                create_entity = False
        else:
            create_entity = False

        if phase is None:
            phase = JobPhase()

        phase.job_id = job.id
        phase.repository_id = job.repository_id
        phase.project_id = job.project_id
        phase.label = stage_type.title()

        phase.date_started = self._get_start_time(stage_list)
        phase.date_finished = self._get_end_time(stage_list)

        # for stage in (s for s in stages if s['status'] == 'failed'):
        if phase.date_started and phase.date_finished:
            if all(s['status'] == 'passed' for s in stage_list):
                phase.result = Result.passed
            else:
                phase.result = Result.failed
            phase.status = Status.finished
        elif phase.date_started:
            if any(s['status'] == 'failed' for s in stage_list):
                phase.result = Result.failed
            else:
                phase.result = Result.unknown
            phase.status = Status.in_progress
        else:
            phase.status = Status.queued
            phase.result = Result.unknown

        if create_entity:
            entity = RemoteEntity(
                provider=self.provider, remote_id=remote_id,
                internal_id=phase.id, type='phase',
            )
            db.session.add(entity)

        db.session.add(phase)

        return phase

    def _sync_step(self, job, phase, stage, step=None):
        if step is None:
            try:
                entity = RemoteEntity.query.filter_by(
                    type='step',
                    provider=self.provider,
                    remote_id=str(stage['id']),
                )[0]
            except IndexError:
                step, entity = None, None
            else:
                step = JobStep.query.get(entity.internal_id)
            create_entity = entity is None
        else:
            create_entity = False

        if step is None:
            step = JobStep()

        node = self._get_node(stage['buildNode'])

        step.job_id = job.id
        step.repository_id = job.repository_id
        step.project_id = job.project_id
        step.phase_id = phase.id
        step.node_id = node.id
        step.label = stage['name']
        step.date_started = self._get_start_time([stage])
        step.date_finished = self._get_end_time([stage])

        if step.date_started and step.date_finished:
            if stage['status'] == 'passed':
                step.result = Result.passed
            else:
                step.result = Result.failed
            step.status = Status.finished
        elif step.date_started:
            if stage['status'] == 'failed':
                step.result = Result.failed
            else:
                step.result = Result.unknown
            step.status = Status.in_progress
        else:
            step.status = Status.queued
            step.result = Result.unknown

        if create_entity:
            entity = RemoteEntity(
                provider=self.provider, remote_id=str(stage['id']),
                internal_id=step.id, type='step',
            )
            db.session.add(entity)

        db.session.add(step)
{'fraction_non_alphanumeric': 0.09373079287031345, 'fraction_numerical': 0.002458512599877074, 'mean_word_length': 1.7631076204627467, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7131424', 'n_tokens_mistral': 3647, 'n_tokens_neox': 3479, 'n_words': 813}