<commit_before># encoding: utf-8
#
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#

from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import re
from copy import copy

import requests

from activedata_etl.transforms import TRY_AGAIN_LATER
from pyLibrary import convert
from pyLibrary.debugs.exceptions import Except
from pyLibrary.debugs.logs import Log, machine_metadata
from pyLibrary.dot import coalesce, wrap, Dict, unwraplist, Null, DictList
from pyLibrary.env import http, elasticsearch
from pyLibrary.maths import Math
from pyLibrary.maths.randoms import Random
from pyLibrary.meta import cache, use_settings
from pyLibrary.queries import jx
from pyLibrary.strings import expand_template
from pyLibrary.thread.threads import Thread, DEBUG
from pyLibrary.times.dates import Date
from pyLibrary.times.durations import HOUR, DAY, MINUTE, SECOND
from pyLibrary.times.timer import Timer

RESULT_SET_URL = "https://treeherder.mozilla.org/api/project/{{branch}}/resultset/?format=json&count=1000&full=true&short_revision__in={{revision}}"
FAILURE_CLASSIFICATION_URL = "https://treeherder.mozilla.org/api/failureclassification/"
REPO_URL = "https://treeherder.mozilla.org:443/api/repository/"
JOBS_URL = "https://treeherder.mozilla.org/api/project/{{branch}}/jobs/?count=2000&offset={{offset}}&result_set_id__in={{result_set_id}}"

DETAILS_URL = "https://treeherder.mozilla.org/api/jobdetail/?job_id__in={{job_id}}&repository={{branch}}"
NOTES_URL = "https://treeherder.mozilla.org/api/project/{{branch}}/note/?job_id={{job_id}}"
JOB_BUG_MAP = "https://treeherder.mozilla.org/api/project/{{branch}}/bug-job-map/?job_id={{job_id}}"


class TreeHerder(object):
    @use_settings
    def __init__(self, hg, use_cache=True, cache=None, rate_limiter=None, settings=None):
        cache.schema = SCHEMA
        rate_limiter.schema = RATE_LIMITER_SCHEMA

        self.settings = settings
        try:
            self.failure_classification = {c.id: c.name for c in http.get_json(FAILURE_CLASSIFICATION_URL)}
            self.repo = {c.id: c.name for c in http.get_json(REPO_URL)}
            self.disabled = False
            Log.warning("Treeherder ingestion is enabled!")
        except Exception:
            self.disabled = True

        self.hg = hg
        self.cache = elasticsearch.Cluster(cache).get_or_create_index(cache)
        self.rate_limiter = elasticsearch.Cluster(cache).get_or_create_index(rate_limiter)
        self.rate_limiter.set_refresh_interval(seconds=1)

    def _get_job_results_from_th(self, branch, revision):
        """
        :param branch:
        :param revision:
        :return:  Null - IF THERE IS NOTHING, RAISE EXCEPTION IF WE SHOULD TRY AGAIN
        """
        start = None
        request = Dict(last=Date.now(), count=0)

        def _register_call(end=None):
            try:
                request.count += 1
                _id = "-".join([branch, revision])
                if not start or end or request.last < Date.now() - 5 * SECOND:
                    request.last = Date.now()
                    response = http.put(
                        url=self.rate_limiter.url + "/" + _id,
                        timeout=3,
                        data=convert.unicode2utf8(convert.value2json(Dict(
                            machine=machine_metadata.name,
                            start=start,
                            last_seen=request.last,
                            end=end,
                            count=request.count
                        )))
                    )
                    if unicode(response.status_code)[0] != '2':
                        Log.error("Could not register call")
            except Exception, e:
                e = Except.wrap(e)
                if "Read timed out." in e or "Caused by ConnectTimeoutError" in e:
                    pass  # IT HAPPENS BECAUSE OF SHORT TIMEOUT, NO NEED TO FREAK OUT, OTHER CALLS WILL FAIL AND INFORM US OF PROBLEMS
                else:
                    Log.warning("Problem registering call", cause=e)

        _register_call()
        start = Date.now().unix

        try:
            url = expand_template(RESULT_SET_URL, {"branch": branch, "revision": revision[0:12:]})
            results = None
            for attempt in range(3):
                try:
                    _register_call()
                    response = http.get(url=url)
                    if str(response.status_code)[0] == b'2':
                        results = convert.json2value(convert.utf82unicode(response.content)).results
                        break
                    elif str(response.status_code)[0] == b'5':
                        # WE MAY HAVE CRUSHED TH
                        Log.error(TRY_AGAIN_LATER, reason="HTTP " + unicode(response.status) + " ERROR")
                    elif response.status_code == 404:
                        if branch not in ["hg.mozilla.org"]:
                            Log.warning("{{branch}} rev {{revision}} returns 404 NOT FOUND", branch=branch, revision=revision)
                        return Null
                    elif response.status_code == 403:
                        Log.error(TRY_AGAIN_LATER, reason="HTTP 403 ERROR")
                    else:
                        Log.warning("Do not know how to deal with TH error {{code}}", code=response.status_code)
                except Exception, e:
                    e = Except.wrap(e)
                    if "No JSON object could be decoded" not in e:
                        Log.error("Could not get good response from {{url}}", url=url, cause=e)

            if results is None:
                Log.error("Could not get good response from {{url}}", url=url, cause=e)

            output = []
            for g, repo_ids in jx.groupby(results.id, size=10):
                repo_ids = wrap(list(repo_ids))
                jobs = DictList()
                with Timer("Get {{num}} jobs", {"num": len(repo_ids)}):
                    while True:
                        _register_call()
                        response = http.get_json(expand_template(JOBS_URL, {"branch": branch, "offset": len(jobs), "result_set_id": ",".join(map(unicode, repo_ids))}))
                        jobs.extend(response.results)
                        if len(response.results) != 2000:
                            break

                with Timer("Get (up to {{num}}) details from TH", {"num": len(jobs)}):
                    details = []
                    for _, ids in jx.groupby(jobs.id, size=40):
                        _register_call()
                        details.extend(http.get_json(
                            url=expand_template(DETAILS_URL, {"branch": branch, "job_id": ",".join(map(unicode, ids))}),
                            retry={"times": 3}
                        ).results)
                    details = {k.job_guid: list(v) for k, v in jx.groupby(details, "job_guid")}

                with Timer("Get (up to {{num}}) stars from TH", {"num": len(jobs)}):
                    stars = []
                    for _, ids in jx.groupby(jobs.id, size=40):
                        _register_call()
                        response = http.get_json(expand_template(JOB_BUG_MAP, {"branch": branch, "job_id": "&job_id=".join(map(unicode, ids))}))
                        stars.extend(response),
                    stars = {k.job_id: list(v) for k, v in jx.groupby(stars, "job_id")}

                with Timer("Get notes from TH"):
                    notes = []
                    for jid in set([j.id for j in jobs if j.failure_classification_id != 1] + stars.keys()):
                        _register_call()<commit_msg>fix debug mode<commit_after># encoding: utf-8
#
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#

from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import re
from copy import copy

import requests

from activedata_etl.transforms import TRY_AGAIN_LATER
from pyLibrary import convert
from pyLibrary.debugs.exceptions import Except
from pyLibrary.debugs.logs import Log, machine_metadata
from pyLibrary.dot import coalesce, wrap, Dict, unwraplist, Null, DictList
from pyLibrary.env import http, elasticsearch
from pyLibrary.maths import Math
from pyLibrary.maths.randoms import Random
from pyLibrary.meta import cache, use_settings
from pyLibrary.queries import jx
from pyLibrary.strings import expand_template
from pyLibrary.thread.threads import Thread
from pyLibrary.times.dates import Date
from pyLibrary.times.durations import HOUR, DAY, MINUTE, SECOND
from pyLibrary.times.timer import Timer

DEBUG = True
RESULT_SET_URL = "https://treeherder.mozilla.org/api/project/{{branch}}/resultset/?format=json&count=1000&full=true&short_revision__in={{revision}}"
FAILURE_CLASSIFICATION_URL = "https://treeherder.mozilla.org/api/failureclassification/"
REPO_URL = "https://treeherder.mozilla.org:443/api/repository/"
JOBS_URL = "https://treeherder.mozilla.org/api/project/{{branch}}/jobs/?count=2000&offset={{offset}}&result_set_id__in={{result_set_id}}"

DETAILS_URL = "https://treeherder.mozilla.org/api/jobdetail/?job_id__in={{job_id}}&repository={{branch}}"
NOTES_URL = "https://treeherder.mozilla.org/api/project/{{branch}}/note/?job_id={{job_id}}"
JOB_BUG_MAP = "https://treeherder.mozilla.org/api/project/{{branch}}/bug-job-map/?job_id={{job_id}}"


class TreeHerder(object):
    @use_settings
    def __init__(self, hg, use_cache=True, cache=None, rate_limiter=None, settings=None):
        cache.schema = SCHEMA
        rate_limiter.schema = RATE_LIMITER_SCHEMA

        self.settings = settings
        try:
            self.failure_classification = {c.id: c.name for c in http.get_json(FAILURE_CLASSIFICATION_URL)}
            self.repo = {c.id: c.name for c in http.get_json(REPO_URL)}
            self.disabled = False
            Log.warning("Treeherder ingestion is enabled!")
        except Exception:
            self.disabled = True

        self.hg = hg
        self.cache = elasticsearch.Cluster(cache).get_or_create_index(cache)
        self.rate_limiter = elasticsearch.Cluster(cache).get_or_create_index(rate_limiter)
        self.rate_limiter.set_refresh_interval(seconds=1)

    def _get_job_results_from_th(self, branch, revision):
        """
        :param branch:
        :param revision:
        :return:  Null - IF THERE IS NOTHING, RAISE EXCEPTION IF WE SHOULD TRY AGAIN
        """
        start = None
        request = Dict(last=Date.now(), count=0)

        def _register_call(end=None):
            try:
                request.count += 1
                _id = "-".join([branch, revision])
                if not start or end or request.last < Date.now() - 5 * SECOND:
                    request.last = Date.now()
                    if DEBUG:
                        Log.note("Call TH")
                    response = http.put(
                        url=self.rate_limiter.url + "/" + _id,
                        timeout=3,
                        data=convert.unicode2utf8(convert.value2json(Dict(
                            machine=machine_metadata.name,
                            start=start,
                            last_seen=request.last,
                            end=end,
                            count=request.count
                        )))
                    )
                    if unicode(response.status_code)[0] != '2':
                        Log.error("Could not register call")
            except Exception, e:
                e = Except.wrap(e)
                if "Read timed out." in e or "Caused by ConnectTimeoutError" in e:
                    pass  # IT HAPPENS BECAUSE OF SHORT TIMEOUT, NO NEED TO FREAK OUT, OTHER CALLS WILL FAIL AND INFORM US OF PROBLEMS
                else:
                    Log.warning("Problem registering call", cause=e)

        _register_call()
        start = Date.now().unix

        try:
            url = expand_template(RESULT_SET_URL, {"branch": branch, "revision": revision[0:12:]})
            results = None
            for attempt in range(3):
                try:
                    _register_call()
                    response = http.get(url=url)
                    if str(response.status_code)[0] == b'2':
                        results = convert.json2value(convert.utf82unicode(response.content)).results
                        break
                    elif str(response.status_code)[0] == b'5':
                        # WE MAY HAVE CRUSHED TH
                        Log.error(TRY_AGAIN_LATER, reason="HTTP " + unicode(response.status) + " ERROR")
                    elif response.status_code == 404:
                        if branch not in ["hg.mozilla.org"]:
                            Log.warning("{{branch}} rev {{revision}} returns 404 NOT FOUND", branch=branch, revision=revision)
                        return Null
                    elif response.status_code == 403:
                        Log.error(TRY_AGAIN_LATER, reason="HTTP 403 ERROR")
                    else:
                        Log.warning("Do not know how to deal with TH error {{code}}", code=response.status_code)
                except Exception, e:
                    e = Except.wrap(e)
                    if "No JSON object could be decoded" not in e:
                        Log.error("Could not get good response from {{url}}", url=url, cause=e)

            if results is None:
                Log.error("Could not get good response from {{url}}", url=url, cause=e)

            output = []
            for g, repo_ids in jx.groupby(results.id, size=10):
                repo_ids = wrap(list(repo_ids))
                jobs = DictList()
                with Timer("Get {{num}} jobs", {"num": len(repo_ids)}, debug=DEBUG):
                    while True:
                        _register_call()
                        response = http.get_json(expand_template(JOBS_URL, {"branch": branch, "offset": len(jobs), "result_set_id": ",".join(map(unicode, repo_ids))}))
                        jobs.extend(response.results)
                        if len(response.results) != 2000:
                            break

                with Timer("Get (up to {{num}}) details from TH", {"num": len(jobs)}, debug=DEBUG):
                    details = []
                    for _, ids in jx.groupby(jobs.id, size=40):
                        _register_call()
                        details.extend(http.get_json(
                            url=expand_template(DETAILS_URL, {"branch": branch, "job_id": ",".join(map(unicode, ids))}),
                            retry={"times": 3}
                        ).results)
                    details = {k.job_guid: list(v) for k, v in jx.groupby(details, "job_guid")}

                with Timer("Get (up to {{num}}) stars from TH", {"num": len(jobs)}, debug=DEBUG):
                    stars = []
                    for _, ids in jx.groupby(jobs.id, size=40):
                        _register_call()
                        response = http.get_json(expand_template(JOB_BUG_MAP, {"branch": branch, "job_id": "&job_id=".join(map(unicode, ids))}))
                        stars.extend(response),
                    stars = {k.job_id: list(v) for k, v in jx.groupby(stars, "job_id")}

                with Timer("Get notes from TH", debug=DEBUG):
                    notes = []
                    for jid in set([j.id for j in jobs if j.failure_classification_id != 1] + stars.keys()):
                        _register_call()
{'fraction_non_alphanumeric': 0.10938580499151317, 'fraction_numerical': 0.007669579430439429, 'mean_word_length': 1.8913122500908761, 'pattern_counts': {'":': 26, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 14, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7218062', 'n_tokens_mistral': 4678, 'n_tokens_neox': 4326, 'n_words': 1084}